перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)

Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-31 15:19:00 +03:00
co-authored by Claude Opus 5.5
parent 276dbdad8a
commit 5112be83cc
70 changed files with 10544 additions and 7 deletions
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
MAKE_PY = "python" if WINDOWS else "python3"
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
import urllib.request
for attempt in range(1, 7):
# The public test stops the replica that served its first POST; its request sequence is
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
# (attempt - 1) extra requests so that the other replica becomes the target as well.
for _ in range(attempt - 1):
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
request straight to S3 (from inside the storage network) is refused, the signed one works.
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import subprocess
import sys
import urllib.error
import urllib.request
import uuid
from email.message import Message
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
PASSWORD = "Demo-password-Ω-12345"
class Client:
def __init__(self, base: str):
self.base = base.rstrip("/")
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
headers: dict | None = None) -> tuple[int, Message, bytes]:
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
h["Content-Type"] = "application/json"
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, r.read() # case-insensitive HTTPMessage
except urllib.error.HTTPError as e:
return e.code, e.headers, e.read()
def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None:
inst = headers.get("X-Instance-Id", "-")
if binary:
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
else:
text = payload.decode("utf-8", "replace")[:300]
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
def user(c: Client, label: str) -> dict:
name = f"{label}_{uuid.uuid4().hex[:8]}"
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
"password": PASSWORD})
assert status == 201, body
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
session = json.loads(body)
return {"id": session["user"]["id"], "token": session["token"], "name": name}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
c = Client(args.base_url)
content = FIXTURE.read_bytes()
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
print(f"users: alice, bob (members), eve (outsider)")
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
print("\n1. upload -> message -> download by the second member")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
raw=content, headers={"Content-Type": "image/png"})
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
att = json.loads(b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id} (bob)", st, h, b)
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
print(f" SHA-256 of the download == fixture == metadata: {same}")
print("\n2. outsider and direct S3 access")
for suffix in ("", "/content"):
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
f"{key not in json.dumps(att)})")
probe = (
"import urllib.request as u, urllib.error as e, asyncio\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
"asyncio.run(m())\n")
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
capture_output=True, text=True).stdout
for line in out.splitlines():
print(" (inside network storage) " + line)
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
return 0 if same else 1
if __name__ == "__main__":
sys.exit(main())
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
python scripts/explain_history.py --output evidence/explain-history.txt
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
1. first page (HISTORY_FIRST)
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
reused by the driver would get it
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
4. the rejected OFFSET pagination for the same late page
Runs psql inside the postgres container; no database port is published.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
CHAT = "00000000-0000-4000-8000-00000000e002"
USER = "00000000-0000-4000-8000-00000000e001"
TARGET = 20_000
LATE = 19_950 # cursor position: the 50 newest messages remain
SEED = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_by, created_at)
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
'{USER}', timestamptz '2026-01-01 00:00:00+00'
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM generate_series(1::int, {TARGET}::int) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
COMMIT;
ANALYZE messages;
"""
def pg(sql_param: str) -> str:
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
parts = sql_param.split("%s")
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
EXPLAIN = f"""
\\set ON_ERROR_STOP on
\\pset pager off
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
SET plan_cache_mode = force_generic_plan;
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
PREPARE or_page(uuid, timestamptz, uuid, int) AS
SELECT id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
PREPARE offset_page(uuid, int, int) AS
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
"""
def psql(script: str) -> str:
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
return result.stdout
def run() -> str:
psql(SEED)
return psql(EXPLAIN)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", default=None)
args = parser.parse_args()
text = run()
print(text)
if args.output:
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
return 0
if __name__ == "__main__":
sys.exit(main())
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
python scripts/init_secrets.py
Idempotent: existing files are kept, so re-running never changes passwords of a
database that already has data. Prints only file names, never values.
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
"""
from __future__ import annotations
import os
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
def write(name: str, value: str) -> None:
path = SECRETS / name
if path.exists():
print(f"keep secrets/{name}")
return
path.write_text(value + "\n", encoding="utf-8", newline="\n")
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
# compose bind mount. Local synthetic lab credentials only.
os.chmod(path, 0o644)
print(f"created secrets/{name}")
def read(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def main() -> int:
SECRETS.mkdir(exist_ok=True)
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
write("valkey_password", secrets.token_urlsafe(24))
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
for who in ("admin", "api", "reconciler"):
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
return 0
if __name__ == "__main__":
sys.exit(main())
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""What the load balancer thinks right now, mapped to real containers.
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
so no admin port is published. Container hostname = X-Instance-Id of that replica.
Prints no secrets. Used by student-tests and bench scripts as a library as well.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
def run(*args: str, timeout: float = 60) -> str:
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
timeout=timeout)
if result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
return result.stdout
def replicas(all_states: bool = True) -> list[dict]:
"""API containers of this compose project (running and stopped)."""
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
if not ids:
return []
data = json.loads(run("docker", "inspect", *ids))
result = []
for item in data:
networks = item["NetworkSettings"]["Networks"] or {}
result.append({
"id": item["Id"][:12],
"name": item["Name"].lstrip("/"),
"hostname": item["Config"]["Hostname"],
"state": item["State"]["Status"],
"health": (item["State"].get("Health") or {}).get("Status", "-"),
"exit_code": item["State"]["ExitCode"],
"started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
})
return sorted(result, key=lambda r: r["name"])
def lb_servers() -> list[dict]:
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
servers = []
for line in raw.splitlines():
parts = line.split()
if len(parts) < 7 or not parts[0].isdigit():
continue
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
if admin & ADMIN_RMAINT or addr == "-":
state = "MAINT"
else:
state = OP_STATE.get(op, str(op))
container = by_ip.get(addr)
servers.append({"slot": name, "addr": addr, "state": state,
"container": container["name"] if container else None,
"instance_id": container["hostname"] if container else None})
return servers
def up_instances() -> set[str]:
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
def wait_up(count: int, timeout: float = 60) -> float:
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
started = time.monotonic()
while True:
try:
if len(up_instances()) == count:
return time.monotonic() - started
except RuntimeError:
pass # lb restarting
if time.monotonic() - started > timeout:
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
time.sleep(0.25)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
sub.add_parser("replicas")
wait = sub.add_parser("wait-up")
wait.add_argument("count", type=int)
wait.add_argument("--timeout", type=float, default=60)
args = parser.parse_args()
if args.command == "status":
for s in lb_servers():
if s["state"] != "MAINT" or s["container"]:
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
print(f"(other slots: MAINT, no container)")
elif args.command == "replicas":
for r in replicas():
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
else:
took = wait_up(args.count, args.timeout)
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
return 0
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
exit /b %ERRORLEVEL%
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
# Output is suppressed by the test; run it by hand to see the steps.
set -eu
cd "$(dirname "$0")/.."
# 1. Entry point and API first: no request is served while the stores restart underneath.
docker compose stop lb api
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
# flush), then start again. The S3 bytes live on the named volume s3data.
docker compose restart postgres valkey s3
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
# the new replica addresses through Docker DNS by itself - no stale backend list.
docker compose start api
docker compose start lb
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --no-recreate --no-deps --wait api
docker compose ps api --format '{{.Name}} {{.Status}}'
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
# 1. build the new image from the working tree;
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
# running old version: expand first, contract in a later release);
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
# the old replicas that were not replaced yet keep running.
set -eu
cd "$(dirname "$0")/.."
PY="${PYTHON:-$(command -v python3 || command -v python)}"
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
export BUILD_REVISION
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
old="$(docker compose ps -q api)"
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
docker compose build api
docker compose run --rm --no-deps migrate
for id in $old; do
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
docker stop --time 15 "$id" >/dev/null
docker rm "$id" >/dev/null
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
echo "rolling-update: replaced $name"
done
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
done
+7
View File
@@ -0,0 +1,7 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
exit /b %ERRORLEVEL%
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
# restore it explicitly afterwards: scripts/restore-api.sh.
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
mode="${STOP_MODE:-stop}"
target=""
# Only RUNNING containers of THIS compose project's service `api` are candidates.
for id in $(docker compose ps -q api); do
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
target="$id"
fi
done
if [ -z "$target" ]; then
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
exit 3
fi
name="$(docker inspect -f '{{.Name}}' "$target")"
case "$mode" in
stop) docker stop --time 15 "$target" >/dev/null ;;
kill) docker stop --time 0 "$target" >/dev/null ;;
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
esac
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"