#!/usr/bin/env python3 """Сценарии уровня 4. Стенд уже запущен. Секреты и токены не печатаются.""" import base64 import hashlib import json import struct import subprocess import sys import time import uuid import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 from image_probe import jpeg_size # noqa: E402 BASE = "http://127.0.0.1:8080" def compose(*args, check_code=True): proc = subprocess.run( ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) if check_code and proc.returncode != 0: raise Failure(proc.stdout[-2000:]) return proc def psql(sql): proc = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql) return proc.stdout.strip() def queues(): proc = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages") return proc.stdout def queue_depth(name): text = queues() for line in text.splitlines(): parts = line.split() if len(parts) >= 2 and parts[0] == name and parts[-1].isdigit(): return int(parts[-1]) raise Failure("queue %s missing:\n%s" % (name, text)) def wait_depth(name, minimum, seconds=20): deadline = time.monotonic() + seconds last = None while time.monotonic() < deadline: try: last = queue_depth(name) except Failure: last = None if last is not None and last >= minimum: return last time.sleep(0.5) raise Failure("queue %s depth %s, want >= %s" % (name, last, minimum)) def wait_status(client, actor, att, wanted, seconds=60): deadline = time.monotonic() + seconds last = att while time.monotonic() < deadline: last = client.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"] if last["status"] == wanted: return last if wanted != "failed" and last["status"] == "failed": raise Failure("attachment failed: " + str(last.get("error_code"))) time.sleep(0.4) raise Failure("status %s did not become %s" % (last.get("status"), wanted)) def wait_postgres_ready(att_id, seconds=70): deadline = time.monotonic() + seconds last = "" while time.monotonic() < deadline: last = psql("SELECT status FROM attachments WHERE id = '%s'" % att_id) if last == "ready": return time.sleep(0.5) raise Failure("postgres status stayed " + last) def register(client): username = "u_g4_" + uuid.uuid4().hex[:12] password = "T3st-" + uuid.uuid4().hex user = client.call( "POST", "/api/v1/users", 201, body={"username": username, "display_name": "Проверка 4", "password": password}, schema="User", )["json"] raw = base64.b64encode((username + ":" + password).encode()).decode() session = client.call( "POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + raw}, schema="Session", )["json"] return {"id": user["id"], "token": session["token"], "user": user} def upload(client, actor, chat_id, filename, content, content_type="image/png"): return client.call( "POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename), 202, actor=actor, raw=content, headers={"Content-Type": content_type}, schema="Attachment", )["json"] def bad_png(): ihdr = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0) chunk = b"IHDR" + ihdr crc = zlib.crc32(chunk) & 0xFFFFFFFF return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc) def webp_size(data): check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP") index = 12 while index + 8 <= len(data): kind = data[index:index + 4] size = int.from_bytes(data[index + 4:index + 8], "little") chunk = data[index + 8:index + 8 + size] if kind == b"VP8X" and len(chunk) >= 10: return (1 + int.from_bytes(chunk[4:7], "little"), 1 + int.from_bytes(chunk[7:10], "little")) if kind == b"VP8 " and len(chunk) >= 10: return (int.from_bytes(chunk[6:8], "little") & 0x3FFF, int.from_bytes(chunk[8:10], "little") & 0x3FFF) if kind == b"VP8L" and len(chunk) >= 5: bits = int.from_bytes(chunk[1:5], "little") return ((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1) index += 8 + size + (size & 1) raise Failure("WebP dimensions not found") def variant_count(att_id): return int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att_id)) def main(): sample = (ROOT / "tests/fixtures/sample.png").read_bytes() client = Client(BASE, None) client.call("GET", "/health/ready", schema="Health") alice = register(client) bob_name = "u_g4b_" + uuid.uuid4().hex[:12] bob_password = "T3st-" + uuid.uuid4().hex bob_user = client.call( "POST", "/api/v1/users", 201, body={"username": bob_name, "display_name": "Боб", "password": bob_password}, schema="User", )["json"] chat = client.call( "POST", "/api/v1/chats", 201, actor=alice, body={"title": "Очередь", "member_ids": [alice["id"], bob_user["id"]]}, schema="Chat", )["json"] notes = [] att = upload(client, alice, chat["id"], "sample.png", sample) check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload snapshot") ready = wait_status(client, alice, att, "ready") names = [item["name"] for item in ready["variants"]] check(names == ["thumbnail", "crop", "optimized"], "variant set " + ",".join(names)) thumb = next(item for item in ready["variants"] if item["name"] == "thumbnail") crop = next(item for item in ready["variants"] if item["name"] == "crop") optimized = next(item for item in ready["variants"] if item["name"] == "optimized") check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail meta") check((crop["width"], crop["height"]) == (128, 128) and crop["content_type"] == "image/jpeg", "crop meta") check(optimized["content_type"] == "image/webp" and (optimized["width"], optimized["height"]) == (320, 200), "webp meta") original = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice) check(original["bytes"] == sample, "original bytes changed") thumb_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=alice) crop_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=alice) webp_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=alice) check(jpeg_size(thumb_bytes["bytes"]) == (256, 160), "thumbnail jpeg") check(jpeg_size(crop_bytes["bytes"]) == (128, 128), "crop jpeg") check(webp_size(webp_bytes["bytes"]) == (320, 200), "webp size") check(hashlib.sha256(thumb_bytes["bytes"]).hexdigest() == thumb["sha256"], "thumb sha") replay = client.call("POST", "/api/v1/attachments/%s/replay" % att["id"], 200, actor=alice, schema="Attachment")["json"] check(replay["status"] == "ready" and replay["variants"] == ready["variants"], "ready replay changed variants") check(variant_count(att["id"]) == 3, "replay created another variant set") check(psql("SELECT outcome FROM job_replays WHERE attachment_id = '%s' ORDER BY created_at DESC LIMIT 1" % att["id"]) == "ignored_ready", "replay audit") notes.append("happy-path ready id=%s variants=3 replay=ignored_ready" % att["id"]) compose("stop", "worker") held = upload(client, alice, chat["id"], "while-worker-down.png", sample) client.call("GET", "/api/v1/attachments/%s/content" % held["id"], 409, actor=alice) client.call( "POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice, body={"text": "очередь не блокирует чат", "attachment_ids": [held["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}, schema="Message", ) before = wait_depth("attachment.process", 1) compose("restart", "rabbitmq") for _ in range(40): probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) if probe.returncode == 0: break time.sleep(1) else: raise Failure("rabbitmq did not come back") after = wait_depth("attachment.process", 1, seconds=30) notes.append("broker restart kept %s then %s process messages" % (before, after)) compose("start", "worker") held_ready = wait_status(client, alice, held, "ready") check(variant_count(held["id"]) == 3, "restarted job duplicated variants") notes.append("worker-down then broker restart kept queue; ready id=%s" % held_ready["id"]) compose("stop", "worker") compose("stop", "rabbitmq") gap = upload(client, alice, chat["id"], "outbox-gap.png", sample) unpublished = psql( "SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id " "WHERE j.attachment_id = '%s' AND o.published_at IS NULL" % gap["id"] ) check(unpublished == "1", "outbox was published while broker was down: " + unpublished) compose("stop", "api") compose("start", "rabbitmq") for _ in range(40): probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) if probe.returncode == 0: break time.sleep(1) compose("start", "worker") wait_postgres_ready(gap["id"]) ready_before_api = psql("SELECT status FROM attachments WHERE id = '%s'" % gap["id"]) check(ready_before_api == "ready", "worker did not recover unpublished outbox") compose("start", "api") for _ in range(40): try: client.call("GET", "/health/ready", schema="Health") break except Failure: time.sleep(1) else: raise Failure("api did not return") gap_body = client.call("GET", "/api/v1/attachments/%s" % gap["id"], actor=alice, schema="Attachment")["json"] check(gap_body["status"] == "ready" and len(gap_body["variants"]) == 3, "gap attachment not ready via api") notes.append("commit/publish gap recovered by worker without a new upload id=%s" % gap["id"]) broken = upload(client, alice, chat["id"], "broken.png", bad_png()) failed = wait_status(client, alice, broken, "failed") check(failed["error_code"] == "invalid_image", "permanent code " + str(failed.get("error_code"))) check(variant_count(broken["id"]) == 0, "failed job stored variants") client.call( "POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, body={"text": "", "attachment_ids": [broken["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}, ) again = client.call("POST", "/api/v1/attachments/%s/replay" % broken["id"], 202, actor=alice, schema="Attachment")["json"] check(again["status"] == "queued", "replay of failed did not queue") failed_again = wait_status(client, alice, broken, "failed") check(failed_again["error_code"] == "invalid_image", "replay did not finish as permanent failure") check(variant_count(broken["id"]) == 0, "replay of failed created variants") dlq = queues() check("attachment.failed" in dlq, "dlq missing") notes.append("permanent invalid_image + replay id=%s dlq present" % broken["id"]) flaky = upload(client, alice, chat["id"], "__lab_fault_transient.png", sample) flaky_ready = wait_status(client, alice, flaky, "ready", seconds=40) failures = psql( "SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND permanent = false AND error_code = 'unavailable'" % flaky["id"] ) check(int(failures) >= 2, "expected two transient failures, got " + failures) check(variant_count(flaky["id"]) == 3, "transient retries duplicated variants") attempt = psql("SELECT attempt FROM attachment_jobs WHERE attachment_id = '%s'" % flaky["id"]) notes.append("backoff retries=%s final_attempt=%s id=%s" % (failures, attempt, flaky_ready["id"])) compose("stop", "worker") crash = upload(client, alice, chat["id"], "crash-before-ack.png", sample) wait_depth("attachment.process", 1) crashed = subprocess.run( ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "run", "--rm", "--no-deps", "--name", "lab5-crash-once", "-e", "CRASH_AFTER_S3=1", "worker"], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=45, ) check(crashed.returncode != 0, "crash hook did not stop the worker") status_after_crash = psql("SELECT status FROM attachments WHERE id = '%s'" % crash["id"]) check(status_after_crash == "processing", "crash did not stop between claim and ready: " + status_after_crash) compose("start", "worker") crash_ready = wait_status(client, alice, crash, "ready") check(variant_count(crash["id"]) == 3, "redelivery created a second variant set") original_after = client.call("GET", "/api/v1/attachments/%s/content" % crash["id"], actor=alice) check(original_after["bytes"] == sample, "redelivery changed the original") staging_left = psql("SELECT staging_key IS NULL FROM attachments WHERE id = '%s'" % crash["id"]) check(staging_left == "t", "staging was not cleared after ready") notes.append("crash after s3 before ack redelivery id=%s variants=3 staging cleared" % crash_ready["id"]) compose("stop", "worker") compose("stop", "rabbitmq") leased = upload(client, alice, chat["id"], "expired-lease.png", sample) psql( "UPDATE attachment_jobs SET status = 'processing', attempt = 1, " "lease_token = gen_random_uuid(), lease_until = now() - interval '2 minutes', " "updated_at = now() WHERE attachment_id = '%s'" % leased["id"] ) psql("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '%s'" % leased["id"]) psql( "UPDATE outbox SET published_at = now() WHERE job_id = " "(SELECT id FROM attachment_jobs WHERE attachment_id = '%s')" % leased["id"] ) compose("start", "rabbitmq") for _ in range(40): probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) if probe.returncode == 0: break time.sleep(1) compose("start", "worker") lease_ready = wait_status(client, alice, leased, "ready") lease_rows = psql( "SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND error_code = 'lease_expired'" % leased["id"] ) check(int(lease_rows) >= 1, "expired lease was not recovered") check(variant_count(leased["id"]) == 3, "lease recovery duplicated variants") notes.append("expired processing lease recovered id=%s" % lease_ready["id"]) text = "\n".join(notes) print(text) evidence = ROOT / "evidence" / "grade4.txt" evidence.write_text(text + "\n", encoding="utf-8") print("PASS: grade 4 fault scenarios") if __name__ == "__main__": try: main() except (Failure, OSError, subprocess.SubprocessError) as exc: print("FAIL:", exc) raise SystemExit(1)