From 962adbe03b41ebb2832e078b63cd5e4b5bdb3171 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:48:26 +0300 Subject: [PATCH] =?UTF-8?q?lab5=20=D1=83=D1=80=D0=BE=D0=B2=D0=B5=D0=BD?= =?UTF-8?q?=D1=8C=204:=20jobctl=20(=D1=80=D0=B5=D0=B5=D1=81=D1=82=D1=80=20?= =?UTF-8?q?failed,=20replay,=20reprocess),=20=D1=82=D0=B5=D1=81=D1=82?= =?UTF-8?q?=D1=8B=20=D0=BE=D0=BA=D0=BE=D0=BD=20=D0=BE=D1=82=D0=BA=D0=B0?= =?UTF-8?q?=D0=B7=D0=B0=20=D0=B8=20demo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 jobctl: replay только failed с живым источником (новое задание replay_of + аудит), reprocess только ready и только более новой версии; решения под той же блокировкой строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий. --- scripts/demo_attachments.py | 19 +- scripts/demo_pipeline.py | 130 ++++++++ src/messenger/jobctl.py | 220 ++++++++++++ student-tests/test_pipeline_flow.py | 500 ++++++++++++++++++++++++++++ 4 files changed, 865 insertions(+), 4 deletions(-) create mode 100644 scripts/demo_pipeline.py create mode 100644 src/messenger/jobctl.py create mode 100644 student-tests/test_pipeline_flow.py diff --git a/scripts/demo_attachments.py b/scripts/demo_attachments.py index 21f98c2..2e24432 100644 --- a/scripts/demo_attachments.py +++ b/scripts/demo_attachments.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids, +"""Lab 4 defense scenario (kept in lab 5; upload is now 202 queued, the download waits for ready): +upload the fixture, send a message with attachment_ids, download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated request straight to S3 (from inside the storage network) is refused, the signed one works. @@ -67,6 +68,16 @@ def user(c: Client, label: str) -> dict: return {"id": session["user"]["id"], "token": session["token"], "name": name} +def wait_ready(c: Client, actor: dict, attachment_id: str, timeout: float = 60): + import time + deadline = time.monotonic() + timeout + while True: + st, h, b = c.call("GET", f"/api/v1/attachments/{attachment_id}", token=actor["token"]) + if st != 200 or json.loads(b)["status"] in ("ready", "failed") or time.monotonic() > deadline: + return st, h, b + time.sleep(0.3) + + def main() -> int: p = argparse.ArgumentParser() p.add_argument("--base-url", default="http://127.0.0.1:8080") @@ -77,7 +88,7 @@ def main() -> int: alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve") print(f"users: alice, bob (members), eve (outsider)") status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"], - body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]}) + body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]}) chat = json.loads(body) print("\n1. upload -> message -> download by the second member") @@ -88,8 +99,8 @@ def main() -> int: st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"], body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b) - st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"]) - show("GET /attachments/{id} (bob)", st, h, b) + st, h, b = wait_ready(c, bob, att["id"]) + show("GET /attachments/{id} (bob) until ready", st, h, b) st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"]) show("GET /attachments/{id}/content (bob)", st, h, got, binary=True) same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"] diff --git a/scripts/demo_pipeline.py b/scripts/demo_pipeline.py new file mode 100644 index 0000000..d2f848c --- /dev/null +++ b/scripts/demo_pipeline.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Lab 5 defense scenario, steps 2-3 (README): worker stopped -> upload 202 -> content 409 while the +rest of the API works -> a message with the attachment is created -> API and broker restarted with +their volumes -> worker started -> the same attachment becomes ready, thumbnail downloadable. + + python scripts/demo_pipeline.py [--base-url http://127.0.0.1:8080] + +Prints every step with HTTP codes, IDs, the outbox row and the queue depth; never prints tokens or +keys. Creates synthetic users. Always starts the worker again at the end. +""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from demo_attachments import FIXTURE, ROOT, Client, show, user, wait_ready # noqa: E402 + + +def compose(*args: str, check: bool = True) -> str: + return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8", + check=check, timeout=300).stdout + + +def psql(sql: str) -> str: + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql).strip() + + +def queue_depth() -> str: + out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers") + line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t")) + _, messages, consumers = line.split("\t") + return f"messages={messages} consumers={consumers}" + + +def wait_api(c: Client, timeout: float = 120) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + if c.call("GET", "/health/ready")[0] == 200: + return + except OSError: + pass + time.sleep(1) + raise SystemExit("API did not become ready") + + +def step(text: str) -> None: + print(f"\n[{time.strftime('%H:%M:%S')}] {text}") + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://127.0.0.1:8080") + args = p.parse_args() + c = Client(args.base_url) + content = FIXTURE.read_bytes() + alice, bob = user(c, "alice"), user(c, "bob") + _, _, body = c.call("POST", "/api/v1/chats", token=alice["token"], + body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]}) + chat = json.loads(body) + try: + step("1. docker compose stop worker") + compose("stop", "worker") + print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip()) + + step("2. upload while the worker is stopped") + st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"], + raw=content, headers={"Content-Type": "image/png"}) + show("POST /chats/{id}/attachments", st, h, b) + att = json.loads(b) + print(f" attachment_id = {att['id']} status = {att['status']}") + for query in ("", "?variant=thumbnail"): + st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}/content{query}", token=bob["token"]) + show(f"GET /attachments/{{id}}/content{query}", st, h, b) + st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=bob["token"], + body={"text": "обычное сообщение работает"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + show("POST /messages (text only)", st, h, b) + st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"], + body={"text": "фото", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + show("POST /messages attachment_ids=[queued id]", st, h, b) + message_id = json.loads(b)["id"] + time.sleep(2) + print(" job + outbox in PostgreSQL: " + psql( + "SELECT j.id || ' state=' || j.state || ' attempt=' || j.attempt || ' | outbox published_at=' || " + f"coalesce(o.published_at::text, 'NULL') FROM jobs j JOIN outbox o ON o.job_id = j.id " + f"WHERE j.attachment_id = '{att['id']}'")) + print(" RabbitMQ queue messenger.attachment-jobs: " + queue_depth()) + + step("3. docker compose restart api broker (volumes preserved)") + compose("restart", "api", "broker") + wait_api(c) + time.sleep(3) + print(" RabbitMQ queue after the restart: " + queue_depth()) + st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"]) + show("GET /attachments/{id}", st, h, b) + finally: + step("4. docker compose start worker") + compose("start", "worker") + st, h, b = wait_ready(c, bob, att["id"], timeout=90) + show("GET /attachments/{id} until ready", st, h, b) + ready = json.loads(b) + thumb = next(v for v in ready["variants"] if v["name"] == "thumbnail") + st, h, data = c.call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=bob["token"]) + show("GET /content?variant=thumbnail", st, h, data, binary=True) + from PIL import Image + size = Image.open(io.BytesIO(data)).size + ok = (st == 200 and hashlib.sha256(data).hexdigest() == thumb["sha256"] and size == (256, 160) + and ready["status"] == "ready") + print(f" thumbnail: real JPEG {size[0]}x{size[1]}, sha256 matches metadata: {ok}") + st, h, orig = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"]) + print(f" original unchanged byte for byte: {orig == content}") + print(f" message {message_id} still references it: " + f"{psql(f'SELECT attachment_ids FROM messages WHERE id = {chr(39)}{message_id}{chr(39)}')}") + step("5. job history (jobctl show)") + shown = json.loads(compose("run", "--rm", "--no-deps", "-T", "jobctl", "show", att["id"])) + for e in shown["events"]: + print(f" {e['at']} {e['event']:16} attempt={e['attempt']} actor={e['actor']}") + return 0 if ok and orig == content else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/jobctl.py b/src/messenger/jobctl.py new file mode 100644 index 0000000..a037e19 --- /dev/null +++ b/src/messenger/jobctl.py @@ -0,0 +1,220 @@ +"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`). + + list-failed [--limit N] failed registry (view failed_jobs), newest first + show attachment state, its jobs, attempts and audit events + stats jobs by state, outbox backlog + replay --operator NAME --reason TEXT + failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt + budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source + still exists (staging copy present or original published). "Repeat" never means + "run a finished job again": a ready attachment is refused (already_ready). + reprocess --pipeline N --operator NAME --reason TEXT + Track 2: build a NEWER pipeline version of a READY attachment from its published original. + Refused unless ready, N known, N > published version and no job of version >= N active. + A not-yet-ready upload is refused: its only source is the staging copy that belongs to the + running job, and its result would race with the first publication. + +Every decision is taken under the attachments row lock, the same lock the cleanup takes before it +marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the +row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing). +Output: one JSON document on stdout, without secrets or file names. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import sys +import uuid + +from . import jobs +from .config import ConfigError, Settings +from .db import Database +from .errors import Unavailable +from .pipeline import PIPELINES + +OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") +EXIT_REFUSED = 3 + + +class Refused(Exception): + pass + + +def operator(value: str) -> str: + if not OPERATOR_RE.fullmatch(value): + raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}") + return value + + +def reason(value: str) -> str: + value = value.strip() + if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value): + raise argparse.ArgumentTypeError("reason: 1-200 printable characters") + return value + + +def attachment_id(value: str) -> str: + try: + return str(uuid.UUID(value)) + except ValueError: + raise argparse.ArgumentTypeError("not a UUID") from None + + +async def lock_attachment(conn, attachment: str): + cur = await conn.execute( + "SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments " + "WHERE id = %s FOR UPDATE", (attachment,)) + row = await cur.fetchone() + if row is None or row[0] == "pending": + raise Refused("not_found") + return row + + +async def replay(conn, args, settings: Settings) -> dict: + status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id) + if status == "ready": + raise Refused("already_ready") + if status != "failed": + raise Refused("not_failed:" + status) + if not has_original and staging_state != "present": + raise Refused("source_missing") # staging removed by the retention cleanup + cur = await conn.execute( + "SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' " + "ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,)) + last = await cur.fetchone() + if last is None: + raise Refused("no_failed_job") + cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')", + (args.attachment_id,)) + if await cur.fetchone(): + raise Refused("active_job") + await conn.execute( + "UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() " + "WHERE id = %s", (args.attachment_id,)) + actor = "jobctl:" + args.operator + await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason) + job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor, + replay_of=last[0], reason=args.reason) + return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]} + + +async def reprocess(conn, args, settings: Settings) -> dict: + status, _, _, published = await lock_attachment(conn, args.attachment_id) + if args.pipeline not in PIPELINES: + raise Refused("unknown_pipeline") + if status != "ready": + raise Refused("not_ready:" + status) + if args.pipeline <= published: + raise Refused("not_newer") + cur = await conn.execute( + "SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')", + (args.attachment_id,)) + (active,) = await cur.fetchone() + if active is not None and active >= args.pipeline: + raise Refused("newer_or_same_active") + job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts, + "jobctl:" + args.operator, reason=args.reason) + return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline, + "published_version": published} + + +async def list_failed(conn, args, settings: Settings) -> dict: + cur = await conn.execute( + "SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, " + "attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,)) + cols = [d.name for d in cur.description] + return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]} + + +async def show(conn, args, settings: Settings) -> dict: + cur = await conn.execute( + "SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, " + "updated_at FROM attachments WHERE id = %s", (args.attachment_id,)) + row = await cur.fetchone() + if row is None: + raise Refused("not_found") + attachment = dict(zip([d.name for d in cur.description], row)) + + async def rows(sql: str) -> list[dict]: + c = await conn.execute(sql, (args.attachment_id,)) + names = [d.name for d in c.description] + return [dict(zip(names, r)) for r in await c.fetchall()] + + return { + "attachment": attachment, + "jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, " + "created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"), + "attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, " + "a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a " + "JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"), + "variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, " + "encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s " + "ORDER BY pipeline_version, name"), + "events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail " + "FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s " + "ORDER BY e.id"), + } + + +async def stats(conn, args, settings: Settings) -> dict: + cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state") + by_state = {r[0]: r[1] for r in await cur.fetchall()} + cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) " + "FROM outbox WHERE published_at IS NULL") + pending, due = await cur.fetchone() + return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due} + + +COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats} + + +def arguments(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess") + sub = p.add_subparsers(dest="command", required=True) + for name in ("replay", "reprocess"): + s = sub.add_parser(name) + s.add_argument("attachment_id", type=attachment_id) + if name == "reprocess": + s.add_argument("--pipeline", type=int, required=True) + s.add_argument("--operator", type=operator, required=True) + s.add_argument("--reason", type=reason, required=True) + s = sub.add_parser("list-failed") + s.add_argument("--limit", type=int, default=50) + s = sub.add_parser("show") + s.add_argument("attachment_id", type=attachment_id) + sub.add_parser("stats") + return p.parse_args(argv) + + +async def run(args: argparse.Namespace) -> int: + settings = Settings.from_env(s3_credentials=False) + db = Database(settings, name="messenger-jobctl") + await db.open() + try: + writes = args.command in ("replay", "reprocess") + result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10) + print(json.dumps(result, ensure_ascii=False, indent=1, default=str)) + return 0 if not writes or result.get("result") == "queued" else 1 + except Refused as exc: + print(json.dumps({"result": "refused", "reason": str(exc)})) + return EXIT_REFUSED + except Unavailable: + print(json.dumps({"result": "error", "reason": "database_unavailable"})) + return 1 + finally: + await db.close() + + +def main(argv: list[str] | None = None) -> int: + args = arguments(argv) + try: + return asyncio.run(run(args)) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/test_pipeline_flow.py b/student-tests/test_pipeline_flow.py new file mode 100644 index 0000000..681fe0b --- /dev/null +++ b/student-tests/test_pipeline_flow.py @@ -0,0 +1,500 @@ +"""Lab 5 pipeline against the running stand: 202 -> queued -> ready, variants by the contract, +the failure windows of contracts/attachment-states.md, retries, the failed registry and replay, +lease expiry of a hung worker, cleanup that never deletes accepted work. + +Failures are injected with the real mechanisms - docker stop/pause/restart of a service, and +FAULT_POINTS of ONE service recreated with an override (conftest.service_with_env) - and every +test restores the normal stand in `finally`. Each test checks exact HTTP codes AND the rows in +PostgreSQL AND the objects in S3, then the read-only invariant audit (I1-I7) of the whole stand.""" +from __future__ import annotations + +import hashlib +import io +import json +import subprocess +import time +import uuid +from pathlib import Path + +import pytest +from PIL import Image + +import imagegen +from conftest import (attachment_row, audit, compose, events_of, job_rows, jobctl, psql, queue_depth, s3_keys, + s3_size, service_down, service_with_env, wait_healthy, wait_stand) + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +FAST_LEASE = {"JOB_LEASE_SECONDS": "6"} # the reaper notices a dead attempt within ~11 s + + +@pytest.fixture() +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +def wait_for(condition, timeout: float, what: str): + deadline = time.monotonic() + timeout + while True: + value = condition() + if value: + return value + assert time.monotonic() < deadline, what + time.sleep(0.5) + + +def check_variants(api, actor, att: dict, source: bytes) -> dict[str, bytes]: + """Download every variant: real format and size decoded with Pillow, size/sha256 = metadata.""" + expected = {"thumbnail": ("image/jpeg", "JPEG"), "crop": ("image/jpeg", "JPEG"), "optimized": ("image/webp", "WEBP")} + assert [v["name"] for v in att["variants"]] == list(expected) + src = Image.open(io.BytesIO(source)) + w, h = src.size + out = {} + for v in att["variants"]: + got = api.content(actor, att["id"], v["name"]) + assert got.headers["content-type"] == expected[v["name"]][0] == v["content_type"] + assert len(got.content) == v["size_bytes"] and hashlib.sha256(got.content).hexdigest() == v["sha256"] + img = Image.open(io.BytesIO(got.content)) + assert img.format == expected[v["name"]][1] and img.size == (v["width"], v["height"]) + out[v["name"]] = got.content + def fit(box): + if w <= box and h <= box: + return w, h + return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box) + sizes = {v["name"]: (v["width"], v["height"]) for v in att["variants"]} + assert sizes == {"thumbnail": fit(256), "crop": (128, 128), "optimized": fit(1280)} + original = api.content(actor, att["id"]) + assert original.content == source # the original is published byte for byte + return out + + +def published_keys(attachment_id: str) -> list[str]: + return psql(f"SELECT object_key FROM attachment_variants WHERE attachment_id = '{attachment_id}' " + f"UNION ALL SELECT object_key FROM attachments WHERE id = '{attachment_id}' ORDER BY 1").split() + + +def attempts(attachment_id: str) -> list[dict]: + out = psql("SELECT coalesce(json_agg(json_build_object('attempt', a.attempt, 'outcome', a.outcome, " + "'error_code', a.error_code, 'token', a.token, 'cleaned', a.objects_cleaned_at IS NOT NULL, " + "'keys', a.planned_keys) ORDER BY a.started_at), '[]') FROM job_attempts a JOIN jobs j ON j.id = a.job_id " + f"WHERE j.attachment_id = '{attachment_id}'") + return json.loads(out) + + +def outbox_of(attachment_id: str) -> list[dict]: + out = psql("SELECT coalesce(json_agg(json_build_object('attempt', o.attempt, 'published', o.published_at IS NOT NULL) " + f"ORDER BY o.id), '[]') FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}'") + return json.loads(out) + + +def names(events: list[dict]) -> list[str]: + return [e["event"] for e in events] + + +# ------------------------------------------------------------------ level 3: the happy path +def test_upload_is_202_queued_then_ready_with_contract_variants(api, pair): + alice, bob, chat = pair + data = imagegen.png(900, 600, noise=True) + response = api.upload(alice, chat, data) + assert response.status_code == 202 + att = response.json() + assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None + ready = api.wait_status(bob, att["id"], "ready") + assert ready["error_code"] is None and ready["sha256"] == hashlib.sha256(data).hexdigest() + check_variants(api, bob, ready, data) + row = attachment_row(att["id"]) + assert row["published_version"] == 1 and row["staging_state"] == "deleted" # staging cleaned after ready + assert s3_size(row["staging_key"], "staging") is None + [job] = job_rows(att["id"]) + assert job["state"] == "succeeded" and job["attempt"] == 1 and job["kind"] == "upload" + assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"] + + +def test_queued_attachment_is_attachable_failed_is_not(api, pair): + alice, bob, chat = pair + with service_down("worker"): # stays queued + att = api.upload(alice, chat, SAMPLE).json() + msg = api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}).json() + assert msg["attachment_ids"] == [att["id"]] + api.wait_status(alice, att["id"], "ready") + with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"): + bad = api.upload(alice, chat, SAMPLE).json() + failed = api.wait_status(bob, bad["id"], "failed") + assert failed["error_code"] == "invalid_image" and failed["variants"] == [] + api.post_message(alice, chat, {"text": "", "attachment_ids": [bad["id"]]}, 409) + api.content(bob, bad["id"], expected=409) + api.content(bob, bad["id"], "thumbnail", expected=409) + + +def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready(api, pair): + """The defence scenario: worker stopped -> upload 202 -> content 409, other messages keep working + -> the message with the attachment is created -> API and broker restarted WITH volumes -> + the job is still in the queue -> worker started -> ready, thumbnail downloadable.""" + alice, bob, chat = pair + compose("stop", "worker") + try: + response = api.upload(alice, chat, SAMPLE) + att = response.json() + assert response.status_code == 202 and att["status"] == "queued" + for variant in (None, "thumbnail"): + assert api.content(bob, att["id"], variant, expected=409).json()["error"]["code"] == "conflict" + api.send(bob, chat, "текст работает без worker") + msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json() + wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job") + assert queue_depth() >= 1 # durable message waits in the queue + compose("restart", "api", "broker") # volumes preserved + wait_healthy("api", "broker", "relay") + wait_stand() + assert queue_depth() >= 1, "the accepted job was lost by the broker restart" + assert api.attachment(bob, att["id"])["status"] == "queued" + finally: + compose("start", "worker") + wait_healthy("worker") + ready = api.wait_status(bob, att["id"], "ready") + check_variants(api, bob, ready, SAMPLE) + items, _ = api.history(bob, chat, 50) + assert any(m["id"] == msg["id"] and m["attachment_ids"] == [att["id"]] for m in items) + [job] = job_rows(att["id"]) + assert job["attempt"] == 1 # no duplicate work + + +# ------------------------------------------------------------------ failure windows +def test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox(api, pair): + """Window "after COMMIT, before publish": the broker is down, the API still answers 202 (it never + talks to the broker), the outbox row stays unpublished. No manual action: when the broker is + back the relay publishes it and the worker finishes the job.""" + alice, bob, chat = pair + with service_down("broker"): + att = api.upload(alice, chat, SAMPLE).json() + assert att["status"] == "queued" + time.sleep(3) + assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}] + api.send(alice, chat, "сообщения работают без брокера") + api.wait_status(bob, att["id"], "ready", timeout=90) + assert outbox_of(att["id"]) == [{"attempt": 1, "published": True}] + assert [j["attempt"] for j in job_rows(att["id"])] == [1] + + +def test_relay_down_job_waits_in_outbox_not_lost(api, pair): + alice, bob, chat = pair + with service_down("relay"): + att = api.upload(alice, chat, SAMPLE).json() + time.sleep(3) + assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}] + assert api.attachment(bob, att["id"])["status"] == "queued" + api.wait_status(bob, att["id"], "ready") + + +def test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate(api, pair): + """Window "after publish, before confirm": the relay dies after the broker confirmed the message + but before outbox.published_at is committed. The restarted relay publishes it AGAIN; the + duplicate is a no-op (one claim, one set of variants).""" + alice, bob, chat = pair + with service_with_env("relay", RELAY_FAULT_POINTS="crash_after_publish"): + att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(bob, att["id"], "ready", timeout=90) + wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 30, "outbox not marked after restart") + events = events_of(att["id"]) + assert names(events).count("fault_injected") == 1 and names(events).count("claimed") == 1 + assert [j["attempt"] for j in job_rows(att["id"])] == [1] + assert len(published_keys(att["id"])) == 4 # original + 3 variants, once + + +def test_duplicate_and_reordered_deliveries_are_noops(api, pair): + alice, bob, chat = pair + with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish,reverse_order"): + atts = [api.upload(alice, chat, imagegen.png(50 + i, 40)).json() for i in range(4)] + for att in atts: + api.wait_status(bob, att["id"], "ready") + time.sleep(2) # let the duplicates be consumed + for att in atts: + assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"] + assert len(published_keys(att["id"])) == 4 + logs = compose("logs", "--no-log-prefix", "worker").stdout + assert '"noop_finished"' in logs or '"noop_leased"' in logs # duplicates were seen and acked + + +def test_worker_crash_before_side_effect_lease_expires_and_job_completes(api, pair): + """The worker dies right after the claim. The redelivered message finds a valid lease (no-op); + the reaper returns the job to queued when the lease expires; attempt 2 publishes.""" + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_claim", **FAST_LEASE): + att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(bob, att["id"], "ready", timeout=90) + ev = names(events_of(att["id"])) + assert ev.count("fault_injected") == 1 and "lease_expired" in ev and ev[-1] == "succeeded" + tries = attempts(att["id"]) + assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")] + assert tries[0]["keys"] and all(s3_size(k) is None for k in tries[0]["keys"]) # nothing written + + +def test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates(api, pair): + """Window "after the S3 write, before the result COMMIT": objects of attempt 1 exist under ITS keys + but are never published; attempt 2 publishes its own; the cleanup deletes attempt 1's objects + after ORPHAN_GRACE_SECONDS (they can never become published: the token is fenced).""" + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_s3_write", ORPHAN_GRACE_SECONDS="2", + CLEANUP_INTERVAL_SECONDS="1", **FAST_LEASE): + att = api.upload(alice, chat, SAMPLE).json() + ready = api.wait_status(bob, att["id"], "ready", timeout=90) + first, second = attempts(att["id"]) + assert (first["outcome"], second["outcome"]) == ("abandoned", "succeeded") + assert set(published_keys(att["id"])) == set(second["keys"]) + wait_for(lambda: attempts(att["id"])[0]["cleaned"], 30, "orphans of attempt 1 not cleaned") + assert all(s3_size(k) is None for k in first["keys"]) + assert all(s3_size(k) is not None for k in second["keys"]) + assert s3_keys(f"a/{att['id']}/") == set(second["keys"]) # exactly one set of objects + check_variants(api, bob, ready, SAMPLE) + + +def test_worker_crash_after_commit_before_ack_redelivery_is_noop(api, pair): + """Window "after the result COMMIT, before the ack": the message comes back; the job is + succeeded -> ack without any work; the same result stays published.""" + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_commit"): + att = api.upload(alice, chat, SAMPLE).json() + ready = api.wait_status(bob, att["id"], "ready", timeout=60) + before = published_keys(att["id"]) + wait_healthy("worker") + wait_for(lambda: '"noop_finished"' in compose("logs", "--no-log-prefix", "worker").stdout, 30, + "the redelivered message was not acknowledged as a no-op") + ev = names(events_of(att["id"])) + assert ev == ["created", "claimed", "fault_injected", "succeeded"] or ev == ["created", "claimed", "succeeded", "fault_injected"] + assert published_keys(att["id"]) == before and [a["outcome"] for a in attempts(att["id"])] == ["succeeded"] + assert api.attachment(bob, att["id"]) == ready + + +# ------------------------------------------------------------------ retries and failures +def test_transient_errors_are_retried_with_jittered_backoff(api, pair): + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="transient_error=2", JOB_RETRY_BASE_SECONDS="1", + JOB_RETRY_MAX_SECONDS="8"): + att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(bob, att["id"], "ready", timeout=60) + events = events_of(att["id"]) + retries = [e for e in events if e["event"] == "retry_scheduled"] + assert [(e["attempt"], e["error_code"]) for e in retries] == [(1, "injected_transient"), (2, "injected_transient")] + for e in retries: + assert 0 <= e["detail"]["delay_s"] <= 1 * 2 ** (e["attempt"] - 1) # full jitter window + assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "retry"), (2, "retry"), (3, "succeeded")] + assert [o["attempt"] for o in outbox_of(att["id"])] == [1, 2, 3] # every retry went through the outbox + + +def test_retry_exhausted_failed_registry_then_replay(api, pair): + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="transient_error", JOB_RETRY_BASE_SECONDS="0.2", + JOB_RETRY_MAX_SECONDS="0.5"): + att = api.upload(alice, chat, SAMPLE).json() + failed = api.wait_status(bob, att["id"], "failed", timeout=60) + assert failed["error_code"] == "retry_exhausted" and failed["variants"] == [] + [job] = job_rows(att["id"]) + assert (job["state"], job["attempt"], job["error_code"]) == ("failed", 5, "retry_exhausted") # JOB_MAX_ATTEMPTS + registry = jobctl("list-failed", "--limit", "200")["failed"] + assert any(r["attachment_id"] == att["id"] and r["error_code"] == "retry_exhausted" for r in registry) + assert attachment_row(att["id"])["staging_state"] == "present" # kept for the replay + # Operator replay: a NEW job (replay_of = the failed one), audit with operator and reason. + result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "fault removed") + assert result["result"] == "queued" and result["replay_of"] == job["id"] + ready = api.wait_status(bob, att["id"], "ready", timeout=60) + check_variants(api, bob, ready, SAMPLE) + jobs = job_rows(att["id"]) + assert [(j["kind"], j["state"]) for j in jobs] == [("upload", "failed"), ("replay", "succeeded")] + ev = events_of(att["id"]) + req = next(e for e in ev if e["event"] == "replay_requested") + assert req["actor"] == "jobctl:alice" and req["detail"]["reason"] == "fault removed" + # "Repeat" never re-runs a finished job: a replay of a ready attachment is refused, nothing created. + refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "again", expect_code=3) + assert refused == {"result": "refused", "reason": "already_ready"} + assert len(job_rows(att["id"])) == 2 + + +def test_permanent_error_fails_at_once_without_retries(api, pair): + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"): + att = api.upload(alice, chat, SAMPLE).json() + failed = api.wait_status(bob, att["id"], "failed") + assert failed["error_code"] == "invalid_image" + assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "failed")] + + +def test_memory_and_time_limits_of_the_worker(api, pair): + """40 Mpx PNG (valid, accepted by the API) with IMAGE_MAX_MEMORY_MB=200: the child process cannot + allocate the raster -> image_too_large (permanent); the worker itself keeps running. + JOB_TIMEOUT_SECONDS=3 with a 10 s stall -> processing_timeout (transient, retried).""" + alice, bob, chat = pair + with service_with_env("worker", IMAGE_MAX_MEMORY_MB="200"): + att = api.upload(alice, chat, imagegen.png(8000, 5000, compress_level=9)).json() + failed = api.wait_status(bob, att["id"], "failed", timeout=90) + assert failed["error_code"] == "image_too_large" + ok = api.upload(alice, chat, SAMPLE).json() # the same worker still works + api.wait_status(bob, ok["id"], "ready") + with service_with_env("worker", WORKER_FAULT_POINTS="slow=10", JOB_TIMEOUT_SECONDS="3"): + slow = api.upload(alice, chat, SAMPLE).json() + wait_for(lambda: any(a["error_code"] == "processing_timeout" for a in attempts(slow["id"])), 60, + "no processing_timeout") + api.wait_status(bob, slow["id"], "ready", timeout=120) # normal worker finishes it + + +def test_bad_message_is_dead_lettered_not_processed(api, pair): + """A message that does not match jobs.schema.json (published here with the relay's own broker + user) is rejected by the worker into the DLQ; nothing is processed.""" + before = queue_depth("messenger.attachment-jobs.dlq") + script = ("import asyncio, aio_pika\n" + "async def m():\n" + " url = open('/run/secrets/broker_url').read().strip()\n" + " c = await aio_pika.connect(url)\n" + " ch = await c.channel(publisher_confirms=True)\n" + " ex = await ch.get_exchange('messenger.jobs', ensure=False)\n" + " await ex.publish(aio_pika.Message(b'{\"url\":\"http://evil/x.png\"}'), routing_key='attachment.process.v1')\n" + " await c.close()\n" + "asyncio.run(m())\n") + compose("exec", "-T", "relay", "python", "-c", script) + wait_for(lambda: queue_depth("messenger.attachment-jobs.dlq") == before + 1, 20, "not dead-lettered") + assert '"bad_message"' in compose("logs", "--no-log-prefix", "worker").stdout + + +# ------------------------------------------------------------------ lease, hung worker, shutdown, concurrency +def worker_containers() -> dict[str, str]: + """hostname (= lease_owner) -> container id of every running worker.""" + ids = compose("ps", "-q", "worker").stdout.split() + out = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True, + text=True, check=True).stdout + return dict(line.split() for line in out.splitlines()) + + +def test_hung_worker_loses_its_lease_and_cannot_publish(api, pair): + """Two workers. The one processing the job is frozen (docker pause: no heartbeat). Its lease + expires, the reaper re-queues the job, the other worker publishes it. When the frozen worker + resumes, its result is fenced by the token: the published result does not change.""" + alice, bob, chat = pair + with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST_LEASE): + att = api.upload(alice, chat, SAMPLE).json() + owner = wait_for(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' " + "AND state = 'processing'"), 30, "job never claimed") + frozen = worker_containers()[owner] + subprocess.run(["docker", "pause", frozen], check=True, capture_output=True) + try: + ready = api.wait_status(bob, att["id"], "ready", timeout=120) + published = published_keys(att["id"]) + finally: + subprocess.run(["docker", "unpause", frozen], check=True, capture_output=True) + time.sleep(12) # the resumed attempt finishes its sleep and tries to publish + assert published_keys(att["id"]) == published and api.attachment(bob, att["id"]) == ready + tries = attempts(att["id"]) + assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")] + assert "lease_expired" in names(events_of(att["id"])) + check_variants(api, bob, ready, SAMPLE) + + +def test_graceful_worker_stop_finishes_the_running_attempt(api, pair): + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="slow=5"): + att = api.upload(alice, chat, SAMPLE).json() + wait_for(lambda: api.attachment(bob, att["id"])["status"] == "processing", 30, "never processing") + started = time.monotonic() + compose("stop", "worker") # SIGTERM: drain, not kill + assert time.monotonic() - started < 30 + assert attachment_row(att["id"])["status"] == "ready" # committed before the exit + compose("start", "worker") + wait_healthy("worker") + assert [a["outcome"] for a in attempts(att["id"])] == ["succeeded"] + + +def test_concurrency_is_bounded_by_prefetch(api, pair): + """WORKER_CONCURRENCY=2: never more than 2 jobs processing and 2 unacked messages at once.""" + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="slow=3"): + atts = [api.upload(alice, chat, imagegen.png(30 + i, 30)).json() for i in range(6)] + peak = 0 + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + states = [api.attachment(bob, a["id"])["status"] for a in atts] + peak = max(peak, states.count("processing")) + unacked = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", + "messages_unacknowledged").stdout + for line in unacked.splitlines(): + if line.startswith("messenger.attachment-jobs\t"): + assert int(line.split("\t")[1]) <= 2, line + if all(s == "ready" for s in states): + break + time.sleep(0.3) + assert all(api.attachment(bob, a["id"])["status"] == "ready" for a in atts) + assert peak == 2 + + +# ------------------------------------------------------------------ cleanup never deletes accepted work +def test_queued_staging_is_never_deleted_even_after_retention(api, pair): + """Reviewer's case in another PR: the worker was down longer than the retention and the cleanup + deleted the original of a job that was still queued. Here: the job waits queued (relay down) + while the cleanup runs every second with retention 0 - the staging copy stays, and the + database itself refuses marking it (CHECK attachments_active_has_source).""" + alice, bob, chat = pair + with service_with_env("worker", STAGING_RETENTION_SECONDS="0", CLEANUP_INTERVAL_SECONDS="1"): + with service_down("relay"): + att = api.upload(alice, chat, SAMPLE).json() + time.sleep(5) # several cleanup passes + row = attachment_row(att["id"]) + assert row["status"] == "queued" and row["staging_state"] == "present" + assert s3_size(row["staging_key"], "staging") == len(SAMPLE) + refused = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", + f"UPDATE attachments SET staging_state = 'deleting' WHERE id = '{att['id']}'", + check=False) + assert refused.returncode != 0 and "attachments_active_has_source" in refused.stderr + ready = api.wait_status(bob, att["id"], "ready", timeout=60) + check_variants(api, bob, ready, SAMPLE) + + +def test_retention_deletes_failed_staging_then_replay_is_refused(api, pair): + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error", STAGING_RETENTION_SECONDS="1", + CLEANUP_INTERVAL_SECONDS="1"): + att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(bob, att["id"], "failed") + wait_for(lambda: attachment_row(att["id"])["staging_state"] == "deleted", 30, "staging not cleaned") + assert s3_size(attachment_row(att["id"])["staging_key"], "staging") is None + refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "too late", expect_code=3) + assert refused == {"result": "refused", "reason": "source_missing"} + assert api.attachment(bob, att["id"])["status"] == "failed" and len(job_rows(att["id"])) == 1 + + +def test_cleanup_racing_a_replay_never_deletes_the_replayed_source(api, pair): + """Reviewer's interleaving in another PR: the cleanup READ its candidates (an expired failed + attachment), a replay committed queued, the cleanup went on and deleted the staging copy. + Here the cleanup pauses exactly there (pause_cleanup); the replay commits during the pause; + the cleanup then re-checks under the row lock and skips the row. The replay finishes ready.""" + alice, bob, chat = pair + with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"): + att = api.upload(alice, chat, SAMPLE).json() + api.wait_status(bob, att["id"], "failed") + with service_with_env("worker", WORKER_FAULT_POINTS="pause_cleanup=8", STAGING_RETENTION_SECONDS="1", + CLEANUP_INTERVAL_SECONDS="1"): + wait_for(lambda: '"cleanup_paused"' in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout + and att["id"] in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout, + 30, "cleanup did not pick the failed attachment") + result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "race test") + assert result["result"] == "queued" + ready = api.wait_status(bob, att["id"], "ready", timeout=90) + check_variants(api, bob, ready, SAMPLE) + assert attachment_row(att["id"])["staging_state"] == "deleted" # deleted only AFTER ready + + +def test_reprocess_of_a_not_ready_upload_is_refused(api, pair): + """Reviewer's case in another PR: worker stopped -> upload 202 -> reprocess with pipeline 2 -> + the attachment stayed queued forever. Here the command is refused and v1 completes.""" + alice, bob, chat = pair + with service_down("worker"): + att = api.upload(alice, chat, SAMPLE).json() + refused = jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "alice", "--reason", "x", + expect_code=3) + assert refused == {"result": "refused", "reason": "not_ready:queued"} + assert len(job_rows(att["id"])) == 1 + api.wait_status(bob, att["id"], "ready") + assert attachment_row(att["id"])["published_version"] == 1 + + +def test_zz_invariants_hold_after_all_scenarios(): + report = audit("--verify-checksums") + assert report["objects"]["published"] > 0