forked from mtusi2026/mtusi2026lab5
перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались. Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker добавляются в следующих коммитах). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
276dbdad8a
commit
5112be83cc
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
|
||||
|
||||
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
|
||||
|
||||
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
|
||||
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
|
||||
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
|
||||
container states before/after every hook. Prints no secrets
|
||||
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
|
||||
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
WINDOWS = os.name == "nt"
|
||||
HOOK = ".cmd" if WINDOWS else ".sh"
|
||||
PYTHON = sys.executable
|
||||
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
|
||||
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
|
||||
MAKE_PY = "python" if WINDOWS else "python3"
|
||||
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
|
||||
ENV = {**os.environ, "PYTHONUTF8": "1"}
|
||||
if WINDOWS:
|
||||
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
||||
timeout=timeout, env={**ENV, **(env or {})})
|
||||
took = time.monotonic() - started
|
||||
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
|
||||
|
||||
|
||||
class Log:
|
||||
def __init__(self, name: str, title: str):
|
||||
self.path = EVIDENCE / name
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8").stdout.strip()
|
||||
self.text = f"# {title}\n# {now()}, commit {commit}\n"
|
||||
|
||||
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
|
||||
code, out = sh(cmd, **kwargs)
|
||||
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
|
||||
self.save()
|
||||
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
|
||||
return code
|
||||
|
||||
def note(self, text: str) -> None:
|
||||
self.text += f"\n{text}\n"
|
||||
self.save()
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.write_text(self.text, encoding="utf-8")
|
||||
|
||||
|
||||
def states() -> list[str]:
|
||||
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
||||
rows = []
|
||||
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
|
||||
st = item["State"]
|
||||
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
|
||||
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
|
||||
return sorted(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
make = shutil.which("make", path=ENV["PATH"]) or "make"
|
||||
# 1. Clean clone -> secrets -> build -> start
|
||||
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
|
||||
log.run(["git", "status", "--short", "--branch"])
|
||||
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
|
||||
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
|
||||
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
|
||||
if not (ROOT / ".env").exists():
|
||||
shutil.copy(ROOT / ".env.example", ROOT / ".env")
|
||||
log.note("$ cp .env.example .env")
|
||||
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
|
||||
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
|
||||
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
# venv for validate / own tests
|
||||
if not Path(VENV_PY).exists():
|
||||
log.run([PYTHON, "-m", "venv", ".venv"])
|
||||
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
|
||||
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
|
||||
|
||||
# 2. make check / validate / test
|
||||
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
|
||||
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
|
||||
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
|
||||
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
|
||||
|
||||
# 3. persistence with states before/after
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
|
||||
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
|
||||
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
|
||||
stopped: set[str] = set()
|
||||
import urllib.request
|
||||
for attempt in range(1, 7):
|
||||
# The public test stops the replica that served its first POST; its request sequence is
|
||||
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
|
||||
# (attempt - 1) extra requests so that the other replica becomes the target as well.
|
||||
for _ in range(attempt - 1):
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
|
||||
before = {r for r in states() if "-api-" in r and " running " in r}
|
||||
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
|
||||
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
|
||||
after = states()
|
||||
gone = [r for r in after if "-api-" in r and " exited " in r]
|
||||
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
stopped.update(r.split()[0] for r in gone)
|
||||
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
|
||||
shown="sh scripts/restore-api.sh")
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
if len(stopped) >= 2:
|
||||
break
|
||||
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
|
||||
|
||||
# 5. own tests
|
||||
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
|
||||
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
|
||||
timeout=3600)
|
||||
|
||||
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
|
||||
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
|
||||
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
|
||||
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
|
||||
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
|
||||
print("done:", EVIDENCE)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
|
||||
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
||||
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
||||
|
||||
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from email.message import Message
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
PASSWORD = "Demo-password-Ω-12345"
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base: str):
|
||||
self.base = base.rstrip("/")
|
||||
|
||||
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
|
||||
headers: dict | None = None) -> tuple[int, Message, bytes]:
|
||||
h = dict(headers or {})
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, r.read() # case-insensitive HTTPMessage
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.headers, e.read()
|
||||
|
||||
|
||||
def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None:
|
||||
inst = headers.get("X-Instance-Id", "-")
|
||||
if binary:
|
||||
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
|
||||
else:
|
||||
text = payload.decode("utf-8", "replace")[:300]
|
||||
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
|
||||
|
||||
|
||||
def user(c: Client, label: str) -> dict:
|
||||
name = f"{label}_{uuid.uuid4().hex[:8]}"
|
||||
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
|
||||
"password": PASSWORD})
|
||||
assert status == 201, body
|
||||
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
||||
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
session = json.loads(body)
|
||||
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
|
||||
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
||||
print(f"users: alice, bob (members), eve (outsider)")
|
||||
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
|
||||
print("\n1. upload -> message -> download by the second member")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
|
||||
att = json.loads(b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id} (bob)", st, h, b)
|
||||
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
||||
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
||||
print(f" SHA-256 of the download == fixture == metadata: {same}")
|
||||
|
||||
print("\n2. outsider and direct S3 access")
|
||||
for suffix in ("", "/content"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
|
||||
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
|
||||
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||||
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
|
||||
f"{key not in json.dumps(att)})")
|
||||
probe = (
|
||||
"import urllib.request as u, urllib.error as e, asyncio\n"
|
||||
"def st(url):\n"
|
||||
" try: return u.urlopen(url, timeout=3).status\n"
|
||||
" except e.HTTPError as x: return x.code\n"
|
||||
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
|
||||
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
||||
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
||||
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
|
||||
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
print(" (inside network storage) " + line)
|
||||
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
|
||||
return 0 if same else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
|
||||
|
||||
python scripts/explain_history.py --output evidence/explain-history.txt
|
||||
|
||||
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
|
||||
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
|
||||
1. first page (HISTORY_FIRST)
|
||||
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
|
||||
reused by the driver would get it
|
||||
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
|
||||
4. the rejected OFFSET pagination for the same late page
|
||||
Runs psql inside the postgres container; no database port is published.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
|
||||
|
||||
CHAT = "00000000-0000-4000-8000-00000000e002"
|
||||
USER = "00000000-0000-4000-8000-00000000e001"
|
||||
TARGET = 20_000
|
||||
LATE = 19_950 # cursor position: the 50 newest messages remain
|
||||
|
||||
SEED = f"""
|
||||
\\set ON_ERROR_STOP on
|
||||
BEGIN;
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
|
||||
INSERT INTO chats (id, title, created_by, created_at)
|
||||
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
|
||||
'{USER}', timestamptz '2026-01-01 00:00:00+00'
|
||||
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
|
||||
ON CONFLICT DO NOTHING;
|
||||
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
||||
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
|
||||
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM generate_series(1::int, {TARGET}::int) AS g
|
||||
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
|
||||
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
|
||||
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
|
||||
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
|
||||
COMMIT;
|
||||
ANALYZE messages;
|
||||
"""
|
||||
|
||||
|
||||
def pg(sql_param: str) -> str:
|
||||
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
|
||||
parts = sql_param.split("%s")
|
||||
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
|
||||
|
||||
|
||||
EXPLAIN = f"""
|
||||
\\set ON_ERROR_STOP on
|
||||
\\pset pager off
|
||||
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
|
||||
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
|
||||
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
|
||||
SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
|
||||
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
|
||||
PREPARE or_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
|
||||
PREPARE offset_page(uuid, int, int) AS
|
||||
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
|
||||
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
|
||||
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
|
||||
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
|
||||
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
|
||||
"""
|
||||
|
||||
|
||||
def psql(script: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
|
||||
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def run() -> str:
|
||||
psql(SEED)
|
||||
return psql(EXPLAIN)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--output", default=None)
|
||||
args = parser.parse_args()
|
||||
text = run()
|
||||
print(text)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
|
||||
|
||||
python scripts/init_secrets.py
|
||||
|
||||
Idempotent: existing files are kept, so re-running never changes passwords of a
|
||||
database that already has data. Prints only file names, never values.
|
||||
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print(f"keep secrets/{name}")
|
||||
return
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
|
||||
# compose bind mount. Local synthetic lab credentials only.
|
||||
os.chmod(path, 0o644)
|
||||
print(f"created secrets/{name}")
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
|
||||
write("valkey_password", secrets.token_urlsafe(24))
|
||||
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
|
||||
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
|
||||
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
|
||||
for who in ("admin", "api", "reconciler"):
|
||||
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
|
||||
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""What the load balancer thinks right now, mapped to real containers.
|
||||
|
||||
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
|
||||
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
|
||||
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
|
||||
|
||||
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
|
||||
so no admin port is published. Container hostname = X-Instance-Id of that replica.
|
||||
Prints no secrets. Used by student-tests and bench scripts as a library as well.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
|
||||
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
|
||||
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
|
||||
|
||||
|
||||
def run(*args: str, timeout: float = 60) -> str:
|
||||
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
timeout=timeout)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def replicas(all_states: bool = True) -> list[dict]:
|
||||
"""API containers of this compose project (running and stopped)."""
|
||||
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
|
||||
if not ids:
|
||||
return []
|
||||
data = json.loads(run("docker", "inspect", *ids))
|
||||
result = []
|
||||
for item in data:
|
||||
networks = item["NetworkSettings"]["Networks"] or {}
|
||||
result.append({
|
||||
"id": item["Id"][:12],
|
||||
"name": item["Name"].lstrip("/"),
|
||||
"hostname": item["Config"]["Hostname"],
|
||||
"state": item["State"]["Status"],
|
||||
"health": (item["State"].get("Health") or {}).get("Status", "-"),
|
||||
"exit_code": item["State"]["ExitCode"],
|
||||
"started_at": item["State"]["StartedAt"],
|
||||
"pid": item["State"]["Pid"],
|
||||
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
|
||||
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
|
||||
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
|
||||
})
|
||||
return sorted(result, key=lambda r: r["name"])
|
||||
|
||||
|
||||
def lb_servers() -> list[dict]:
|
||||
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
|
||||
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
|
||||
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
|
||||
servers = []
|
||||
for line in raw.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 7 or not parts[0].isdigit():
|
||||
continue
|
||||
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
|
||||
if admin & ADMIN_RMAINT or addr == "-":
|
||||
state = "MAINT"
|
||||
else:
|
||||
state = OP_STATE.get(op, str(op))
|
||||
container = by_ip.get(addr)
|
||||
servers.append({"slot": name, "addr": addr, "state": state,
|
||||
"container": container["name"] if container else None,
|
||||
"instance_id": container["hostname"] if container else None})
|
||||
return servers
|
||||
|
||||
|
||||
def up_instances() -> set[str]:
|
||||
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
|
||||
|
||||
|
||||
def wait_up(count: int, timeout: float = 60) -> float:
|
||||
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
if len(up_instances()) == count:
|
||||
return time.monotonic() - started
|
||||
except RuntimeError:
|
||||
pass # lb restarting
|
||||
if time.monotonic() - started > timeout:
|
||||
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("status")
|
||||
sub.add_parser("replicas")
|
||||
wait = sub.add_parser("wait-up")
|
||||
wait.add_argument("count", type=int)
|
||||
wait.add_argument("--timeout", type=float, default=60)
|
||||
args = parser.parse_args()
|
||||
if args.command == "status":
|
||||
for s in lb_servers():
|
||||
if s["state"] != "MAINT" or s["container"]:
|
||||
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
|
||||
print(f"(other slots: MAINT, no container)")
|
||||
elif args.command == "replicas":
|
||||
for r in replicas():
|
||||
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
|
||||
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
|
||||
else:
|
||||
took = wait_up(args.count, args.timeout)
|
||||
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
|
||||
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
|
||||
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
|
||||
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
|
||||
# Output is suppressed by the test; run it by hand to see the steps.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# 1. Entry point and API first: no request is served while the stores restart underneath.
|
||||
docker compose stop lb api
|
||||
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
|
||||
# flush), then start again. The S3 bytes live on the named volume s3data.
|
||||
docker compose restart postgres valkey s3
|
||||
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
|
||||
# the new replica addresses through Docker DNS by itself - no stale backend list.
|
||||
docker compose start api
|
||||
docker compose start lb
|
||||
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
|
||||
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
|
||||
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-recreate --no-deps --wait api
|
||||
docker compose ps api --format '{{.Name}} {{.Status}}'
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
|
||||
# 1. build the new image from the working tree;
|
||||
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
|
||||
# running old version: expand first, contract in a later release);
|
||||
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
|
||||
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
|
||||
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
|
||||
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
|
||||
# the old replicas that were not replaced yet keep running.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY="${PYTHON:-$(command -v python3 || command -v python)}"
|
||||
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
|
||||
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
export BUILD_REVISION
|
||||
|
||||
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
|
||||
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
|
||||
old="$(docker compose ps -q api)"
|
||||
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
|
||||
|
||||
docker compose build api
|
||||
docker compose run --rm --no-deps migrate
|
||||
|
||||
for id in $old; do
|
||||
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
|
||||
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
|
||||
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
|
||||
docker stop --time 15 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
|
||||
echo "rolling-update: replaced $name"
|
||||
done
|
||||
for id in $(docker compose ps -q api); do
|
||||
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
|
||||
done
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
|
||||
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
|
||||
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
|
||||
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
|
||||
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
|
||||
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
|
||||
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
|
||||
# restore it explicitly afterwards: scripts/restore-api.sh.
|
||||
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
|
||||
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
|
||||
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
mode="${STOP_MODE:-stop}"
|
||||
|
||||
target=""
|
||||
# Only RUNNING containers of THIS compose project's service `api` are candidates.
|
||||
for id in $(docker compose ps -q api); do
|
||||
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target="$id"
|
||||
fi
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
|
||||
exit 3
|
||||
fi
|
||||
name="$(docker inspect -f '{{.Name}}' "$target")"
|
||||
case "$mode" in
|
||||
stop) docker stop --time 15 "$target" >/dev/null ;;
|
||||
kill) docker stop --time 0 "$target" >/dev/null ;;
|
||||
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
|
||||
esac
|
||||
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"
|
||||
Reference in New Issue
Block a user