#!/usr/bin/env python3 """Grade-4 fault demos. Prints statuses and counts only; never prints tokens or passwords.""" import json import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure # noqa: E402 from image_probe import jpeg_size # noqa: E402 lines = [] def log(message): print(message, flush=True) lines.append(message) def run(args, check=True): result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True) if check and result.returncode != 0: raise SystemExit(args[0] + " failed: " + result.stderr[-400:]) return result def compose(*args): run(["docker", "compose", "--env-file", ".env", *args]) def demo_token(): for line in (ROOT / ".env").read_text().splitlines(): if line.startswith("DEMO_TOKEN="): return line.split("=", 1)[1].strip() return "lab-demo" def psql(sql): result = run([ "docker", "exec", "messenger-lab6-postgres-1", "sh", "-c", 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "$1"', "psql", sql, ]) return result.stdout.strip() def topic_end(topic): result = run([ "docker", "exec", "messenger-lab6-kafka-1", "/opt/kafka/bin/kafka-get-offsets.sh", "--bootstrap-server", "localhost:9092", "--topic", topic, ]) total = 0 for line in result.stdout.splitlines(): parts = line.strip().split(":") if len(parts) >= 3 and parts[-1].lstrip("-").isdigit(): total += int(parts[-1]) return total def offsets(): return "attachments.process end=" + str(topic_end("attachments.process")) + "\nattachments.failed end=" + str(topic_end("attachments.failed")) def restart_count(): result = run(["docker", "inspect", "-f", "{{.RestartCount}}", "messenger-lab6-worker-1"]) return int(result.stdout.strip()) def wait_health(client): deadline = time.monotonic() + 90 while True: try: client.call("GET", "/health/ready", schema="Health") return except Failure: if time.monotonic() > deadline: raise time.sleep(1) def poll(client, actor, attachment, seconds=70): deadline = time.monotonic() + seconds last = attachment while time.monotonic() < deadline: last = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=actor, schema="Attachment")["json"] if last["status"] in ("ready", "failed"): return last time.sleep(0.4) raise SystemExit("attachment stayed " + last["status"]) def summary(attachment): variants = [ f"{item['name']}:{item['content_type']}:{item['width']}x{item['height']}" for item in attachment.get("variants") or [] ] return f"id={attachment['id']} status={attachment['status']} error={attachment.get('error_code')} variants={variants}" def webp_size(data): if data[:4] != b"RIFF" or data[8:12] != b"WEBP": raise SystemExit("optimized payload is not WebP") index = 12 while index + 8 <= len(data): kind = data[index:index + 4] size = int.from_bytes(data[index + 4:index + 8], "little") payload = data[index + 8:index + 8 + size] if kind == b"VP8 ": width = int.from_bytes(payload[6:8], "little") & 0x3FFF height = int.from_bytes(payload[8:10], "little") & 0x3FFF return width, height if kind == b"VP8X": width = 1 + int.from_bytes(payload[4:7], "little") height = 1 + int.from_bytes(payload[7:10], "little") return width, height index += 8 + size + (size & 1) raise SystemExit("WebP dimensions not found") def content(client, actor, attachment, variant=None): path = "/api/v1/attachments/" + attachment["id"] + "/content" if variant: path += "?variant=" + variant return client.call("GET", path, actor=actor) def main(): token = demo_token() client = Client("http://127.0.0.1:8080", timeout=15) wait_health(client) headers = {"X-Demo-Token": token} log("== worker down, API restart, then worker ==") compose("stop", "worker") alice, bob, _eve, chat = client.bootstrap() attachment, original = client.attachment(alice, chat) client.message(alice, chat, "вложение ещё queued", attachments=[attachment["id"]]) blocked = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=bob) log("queued upload " + summary(attachment) + " content=" + blocked["json"]["error"]["code"]) compose("restart", "api") wait_health(client) queued = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=bob, schema="Attachment")["json"] log("after API restart " + summary(queued)) if queued["status"] != "queued": raise SystemExit("job was lost while worker and API were restarted") compose("start", "worker") ready = poll(client, bob, queued) log("after worker start " + summary(ready)) downloaded = content(client, bob, ready) if downloaded["bytes"] != original: raise SystemExit("original bytes changed") thumb = content(client, bob, ready, "thumbnail") crop = content(client, bob, ready, "crop") optimized = content(client, bob, ready, "optimized") log(f"bytes original={len(original)} thumbnail={jpeg_size(thumb['bytes'])} {thumb['headers'].get_content_type()} " f"crop={jpeg_size(crop['bytes'])} {crop['headers'].get_content_type()} " f"optimized={webp_size(optimized['bytes'])} {optimized['headers'].get_content_type()}") log("== broker restart with durable topic ==") compose("stop", "worker") published_before = topic_end("attachments.process") held, _ = client.attachment(alice, chat) deadline = time.monotonic() + 20 published = published_before while time.monotonic() < deadline: published = topic_end("attachments.process") if published > published_before: break time.sleep(0.5) log("topic end before kafka restart: " + str(published_before) + " -> " + str(published)) if published <= published_before: raise SystemExit("accepted job was not waiting in the broker") compose("restart", "kafka") deadline = time.monotonic() + 60 while time.monotonic() < deadline: state = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", "messenger-lab6-kafka-1"], check=False) if state.stdout.strip() == "healthy": break time.sleep(1) retained = topic_end("attachments.process") log("topic end after kafka restart: " + str(retained)) if retained < published: raise SystemExit("broker restart dropped the waiting job") compose("start", "worker") held_ready = poll(client, alice, held) log("after broker restart " + summary(held_ready)) log("== outbox paused across API kill ==") paused = json.loads(client.call("POST", "/internal/demo/outbox-pause", headers=headers)["bytes"]) staged, _ = client.attachment(alice, chat) outbox = json.loads(client.call("GET", "/internal/demo/outbox", headers=headers)["bytes"]) blocked = client.call("GET", "/api/v1/attachments/" + staged["id"] + "/content", 409, actor=alice) log(f"paused={paused['paused']} unpublished={outbox['unpublished']} content={blocked['json']['error']['code']} {summary(staged)}") compose("kill", "api") compose("start", "api") wait_health(client) outbox = json.loads(client.call("GET", "/internal/demo/outbox", headers=headers)["bytes"]) staged = client.call("GET", "/api/v1/attachments/" + staged["id"], actor=alice, schema="Attachment")["json"] log(f"after API kill paused={outbox['paused']} unpublished={outbox['unpublished']} {summary(staged)}") if not outbox["paused"] or outbox["unpublished"] < 1 or staged["status"] != "queued": raise SystemExit("outbox intention did not survive API kill") client.call("POST", "/internal/demo/outbox-resume", headers=headers) staged = poll(client, alice, staged) log("after outbox resume " + summary(staged)) log("== worker exit after object write, before ack ==") before = restart_count() run(["docker", "exec", "messenger-lab6-worker-1", "touch", "/tmp/messenger-crash-after-write"]) crashed, _ = client.attachment(alice, chat) deadline = time.monotonic() + 40 while time.monotonic() < deadline and restart_count() <= before: time.sleep(0.5) after = restart_count() crashed = poll(client, alice, crashed) variants = psql("select name || ':' || count(*) from attachment_variants where attachment_id='" + crashed["id"] + "' group by name order by name") log(f"restart_count {before}->{after} {summary(crashed)} db={variants}") replay = json.loads(client.call("POST", "/internal/demo/attachments/" + crashed["id"] + "/replay", headers=headers)["bytes"]) log("replay of ready status=" + replay["status"] + " variants=" + str(len(replay.get("variants") or []))) log("== permanent failure and explicit replay ==") failed_before = json.loads(client.call("GET", "/internal/demo/failed", headers=headers)["bytes"])["count"] run(["docker", "exec", "messenger-lab6-worker-1", "touch", "/tmp/messenger-fail-permanent"]) broken, _ = client.attachment(alice, chat) broken = poll(client, alice, broken) denied = client.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, body={"text": "нельзя прикрепить failed", "attachment_ids": [broken["id"]]}, headers={"Idempotency-Key": "fail-" + broken["id"]}) failed_after = json.loads(client.call("GET", "/internal/demo/failed", headers=headers)["bytes"])["count"] log(f"failed registry {failed_before}->{failed_after} message={denied['json']['error']['code']} {summary(broken)}") log("topics:\n" + offsets()) replay = json.loads(client.call("POST", "/internal/demo/attachments/" + broken["id"] + "/replay", headers=headers)["bytes"]) log("replay of failed status=" + replay["status"]) recovered = poll(client, alice, replay) variants = psql("select name || ':' || count(*) from attachment_variants where attachment_id='" + recovered["id"] + "' group by name order by name") again = json.loads(client.call("POST", "/internal/demo/attachments/" + recovered["id"] + "/replay", headers=headers)["bytes"]) log(f"after replay {summary(recovered)} db={variants} second_replay={again['status']}") log("== rejected upload ==") rejected = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bad.png", 415, actor=alice, raw=b"not-an-image", headers={"Content-Type": "image/png"}) log("corrupt png " + rejected["json"]["error"]["code"]) log("== runtime limits ==") ps = run(["docker", "ps", "--filter", "name=messenger-lab6", "--format", "{{.Names}} {{.Status}} {{.Ports}}"]) log(ps.stdout.rstrip()) for name in ("api-1", "api-2", "worker-1", "lb-1", "postgres-1", "redis-1", "kafka-1", "s3-1"): inspected = run(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}} {{json .HostConfig.PortBindings}}", "messenger-lab6-" + name]) log(name + " " + inspected.stdout.strip()) anonymous = run(["docker", "exec", "messenger-lab6-api-1", "curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "http://s3:8333/messenger-attachments/originals/missing"]) log("anonymous s3 status=" + anonymous.stdout.strip()) log("PASS grade4 evidence") if __name__ == "__main__": try: main() except (Failure, TimeoutError, OSError) as exc: log("FAIL " + str(exc)) raise SystemExit(1) finally: subprocess.run(["docker", "exec", "messenger-lab6-worker-1", "rm", "-f", "/tmp/messenger-crash-after-write", "/tmp/messenger-fail-permanent"], cwd=ROOT) token = demo_token() subprocess.run(["curl", "-fsS", "-X", "POST", "-H", "X-Demo-Token: " + token, "http://127.0.0.1:8080/internal/demo/outbox-resume"], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.run(["docker", "compose", "--env-file", ".env", "start", "worker", "api"], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) (ROOT / "evidence").mkdir(exist_ok=True) (ROOT / "evidence" / "grade4.txt").write_text("\n".join(lines) + "\n")