lab5 уровень 4: jobctl (реестр failed, replay, reprocess), тесты окон отказа и demo

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

jobctl: replay только failed с живым источником (новое задание replay_of + аудит),
reprocess только ready и только более новой версии; решения под той же блокировкой
строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с
jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший
worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий.
This commit is contained in:
Alexey Bodanyuk
2026-10-31 16:48:26 +03:00
parent 1d37fd0dab
commit 962adbe03b
4 changed files with 865 additions and 4 deletions
+15 -4
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
"""Lab 4 defense scenario (kept in lab 5; upload is now 202 queued, the download waits for ready):
upload the fixture, send a message with attachment_ids,
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
request straight to S3 (from inside the storage network) is refused, the signed one works.
@@ -67,6 +68,16 @@ def user(c: Client, label: str) -> dict:
return {"id": session["user"]["id"], "token": session["token"], "name": name}
def wait_ready(c: Client, actor: dict, attachment_id: str, timeout: float = 60):
import time
deadline = time.monotonic() + timeout
while True:
st, h, b = c.call("GET", f"/api/v1/attachments/{attachment_id}", token=actor["token"])
if st != 200 or json.loads(b)["status"] in ("ready", "failed") or time.monotonic() > deadline:
return st, h, b
time.sleep(0.3)
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
@@ -77,7 +88,7 @@ def main() -> int:
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
print(f"users: alice, bob (members), eve (outsider)")
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
print("\n1. upload -> message -> download by the second member")
@@ -88,8 +99,8 @@ def main() -> int:
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id} (bob)", st, h, b)
st, h, b = wait_ready(c, bob, att["id"])
show("GET /attachments/{id} (bob) until ready", st, h, b)
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Lab 5 defense scenario, steps 2-3 (README): worker stopped -> upload 202 -> content 409 while the
rest of the API works -> a message with the attachment is created -> API and broker restarted with
their volumes -> worker started -> the same attachment becomes ready, thumbnail downloadable.
python scripts/demo_pipeline.py [--base-url http://127.0.0.1:8080]
Prints every step with HTTP codes, IDs, the outbox row and the queue depth; never prints tokens or
keys. Creates synthetic users. Always starts the worker again at the end.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from demo_attachments import FIXTURE, ROOT, Client, show, user, wait_ready # noqa: E402
def compose(*args: str, check: bool = True) -> str:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
check=check, timeout=300).stdout
def psql(sql: str) -> str:
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql).strip()
def queue_depth() -> str:
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
_, messages, consumers = line.split("\t")
return f"messages={messages} consumers={consumers}"
def wait_api(c: Client, timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if c.call("GET", "/health/ready")[0] == 200:
return
except OSError:
pass
time.sleep(1)
raise SystemExit("API did not become ready")
def step(text: str) -> None:
print(f"\n[{time.strftime('%H:%M:%S')}] {text}")
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
c = Client(args.base_url)
content = FIXTURE.read_bytes()
alice, bob = user(c, "alice"), user(c, "bob")
_, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
try:
step("1. docker compose stop worker")
compose("stop", "worker")
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
step("2. upload while the worker is stopped")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
raw=content, headers={"Content-Type": "image/png"})
show("POST /chats/{id}/attachments", st, h, b)
att = json.loads(b)
print(f" attachment_id = {att['id']} status = {att['status']}")
for query in ("", "?variant=thumbnail"):
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}/content{query}", token=bob["token"])
show(f"GET /attachments/{{id}}/content{query}", st, h, b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=bob["token"],
body={"text": "обычное сообщение работает"}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /messages (text only)", st, h, b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "фото", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /messages attachment_ids=[queued id]", st, h, b)
message_id = json.loads(b)["id"]
time.sleep(2)
print(" job + outbox in PostgreSQL: " + psql(
"SELECT j.id || ' state=' || j.state || ' attempt=' || j.attempt || ' | outbox published_at=' || "
f"coalesce(o.published_at::text, 'NULL') FROM jobs j JOIN outbox o ON o.job_id = j.id "
f"WHERE j.attachment_id = '{att['id']}'"))
print(" RabbitMQ queue messenger.attachment-jobs: " + queue_depth())
step("3. docker compose restart api broker (volumes preserved)")
compose("restart", "api", "broker")
wait_api(c)
time.sleep(3)
print(" RabbitMQ queue after the restart: " + queue_depth())
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id}", st, h, b)
finally:
step("4. docker compose start worker")
compose("start", "worker")
st, h, b = wait_ready(c, bob, att["id"], timeout=90)
show("GET /attachments/{id} until ready", st, h, b)
ready = json.loads(b)
thumb = next(v for v in ready["variants"] if v["name"] == "thumbnail")
st, h, data = c.call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=bob["token"])
show("GET /content?variant=thumbnail", st, h, data, binary=True)
from PIL import Image
size = Image.open(io.BytesIO(data)).size
ok = (st == 200 and hashlib.sha256(data).hexdigest() == thumb["sha256"] and size == (256, 160)
and ready["status"] == "ready")
print(f" thumbnail: real JPEG {size[0]}x{size[1]}, sha256 matches metadata: {ok}")
st, h, orig = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
print(f" original unchanged byte for byte: {orig == content}")
print(f" message {message_id} still references it: "
f"{psql(f'SELECT attachment_ids FROM messages WHERE id = {chr(39)}{message_id}{chr(39)}')}")
step("5. job history (jobctl show)")
shown = json.loads(compose("run", "--rm", "--no-deps", "-T", "jobctl", "show", att["id"]))
for e in shown["events"]:
print(f" {e['at']} {e['event']:16} attempt={e['attempt']} actor={e['actor']}")
return 0 if ok and orig == content else 1
if __name__ == "__main__":
sys.exit(main())
+220
View File
@@ -0,0 +1,220 @@
"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`).
list-failed [--limit N] failed registry (view failed_jobs), newest first
show <attachment_id> attachment state, its jobs, attempts and audit events
stats jobs by state, outbox backlog
replay <attachment_id> --operator NAME --reason TEXT
failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt
budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source
still exists (staging copy present or original published). "Repeat" never means
"run a finished job again": a ready attachment is refused (already_ready).
reprocess <attachment_id> --pipeline N --operator NAME --reason TEXT
Track 2: build a NEWER pipeline version of a READY attachment from its published original.
Refused unless ready, N known, N > published version and no job of version >= N active.
A not-yet-ready upload is refused: its only source is the staging copy that belongs to the
running job, and its result would race with the first publication.
Every decision is taken under the attachments row lock, the same lock the cleanup takes before it
marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the
row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing).
Output: one JSON document on stdout, without secrets or file names.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import re
import sys
import uuid
from . import jobs
from .config import ConfigError, Settings
from .db import Database
from .errors import Unavailable
from .pipeline import PIPELINES
OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
EXIT_REFUSED = 3
class Refused(Exception):
pass
def operator(value: str) -> str:
if not OPERATOR_RE.fullmatch(value):
raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}")
return value
def reason(value: str) -> str:
value = value.strip()
if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value):
raise argparse.ArgumentTypeError("reason: 1-200 printable characters")
return value
def attachment_id(value: str) -> str:
try:
return str(uuid.UUID(value))
except ValueError:
raise argparse.ArgumentTypeError("not a UUID") from None
async def lock_attachment(conn, attachment: str):
cur = await conn.execute(
"SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments "
"WHERE id = %s FOR UPDATE", (attachment,))
row = await cur.fetchone()
if row is None or row[0] == "pending":
raise Refused("not_found")
return row
async def replay(conn, args, settings: Settings) -> dict:
status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id)
if status == "ready":
raise Refused("already_ready")
if status != "failed":
raise Refused("not_failed:" + status)
if not has_original and staging_state != "present":
raise Refused("source_missing") # staging removed by the retention cleanup
cur = await conn.execute(
"SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' "
"ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,))
last = await cur.fetchone()
if last is None:
raise Refused("no_failed_job")
cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
(args.attachment_id,))
if await cur.fetchone():
raise Refused("active_job")
await conn.execute(
"UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() "
"WHERE id = %s", (args.attachment_id,))
actor = "jobctl:" + args.operator
await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason)
job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor,
replay_of=last[0], reason=args.reason)
return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]}
async def reprocess(conn, args, settings: Settings) -> dict:
status, _, _, published = await lock_attachment(conn, args.attachment_id)
if args.pipeline not in PIPELINES:
raise Refused("unknown_pipeline")
if status != "ready":
raise Refused("not_ready:" + status)
if args.pipeline <= published:
raise Refused("not_newer")
cur = await conn.execute(
"SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
(args.attachment_id,))
(active,) = await cur.fetchone()
if active is not None and active >= args.pipeline:
raise Refused("newer_or_same_active")
job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts,
"jobctl:" + args.operator, reason=args.reason)
return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline,
"published_version": published}
async def list_failed(conn, args, settings: Settings) -> dict:
cur = await conn.execute(
"SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, "
"attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,))
cols = [d.name for d in cur.description]
return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]}
async def show(conn, args, settings: Settings) -> dict:
cur = await conn.execute(
"SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, "
"updated_at FROM attachments WHERE id = %s", (args.attachment_id,))
row = await cur.fetchone()
if row is None:
raise Refused("not_found")
attachment = dict(zip([d.name for d in cur.description], row))
async def rows(sql: str) -> list[dict]:
c = await conn.execute(sql, (args.attachment_id,))
names = [d.name for d in c.description]
return [dict(zip(names, r)) for r in await c.fetchall()]
return {
"attachment": attachment,
"jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, "
"created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"),
"attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, "
"a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a "
"JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"),
"variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, "
"encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s "
"ORDER BY pipeline_version, name"),
"events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail "
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s "
"ORDER BY e.id"),
}
async def stats(conn, args, settings: Settings) -> dict:
cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state")
by_state = {r[0]: r[1] for r in await cur.fetchall()}
cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) "
"FROM outbox WHERE published_at IS NULL")
pending, due = await cur.fetchone()
return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due}
COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats}
def arguments(argv: list[str] | None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess")
sub = p.add_subparsers(dest="command", required=True)
for name in ("replay", "reprocess"):
s = sub.add_parser(name)
s.add_argument("attachment_id", type=attachment_id)
if name == "reprocess":
s.add_argument("--pipeline", type=int, required=True)
s.add_argument("--operator", type=operator, required=True)
s.add_argument("--reason", type=reason, required=True)
s = sub.add_parser("list-failed")
s.add_argument("--limit", type=int, default=50)
s = sub.add_parser("show")
s.add_argument("attachment_id", type=attachment_id)
sub.add_parser("stats")
return p.parse_args(argv)
async def run(args: argparse.Namespace) -> int:
settings = Settings.from_env(s3_credentials=False)
db = Database(settings, name="messenger-jobctl")
await db.open()
try:
writes = args.command in ("replay", "reprocess")
result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10)
print(json.dumps(result, ensure_ascii=False, indent=1, default=str))
return 0 if not writes or result.get("result") == "queued" else 1
except Refused as exc:
print(json.dumps({"result": "refused", "reason": str(exc)}))
return EXIT_REFUSED
except Unavailable:
print(json.dumps({"result": "error", "reason": "database_unavailable"}))
return 1
finally:
await db.close()
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+500
View File
@@ -0,0 +1,500 @@
"""Lab 5 pipeline against the running stand: 202 -> queued -> ready, variants by the contract,
the failure windows of contracts/attachment-states.md, retries, the failed registry and replay,
lease expiry of a hung worker, cleanup that never deletes accepted work.
Failures are injected with the real mechanisms - docker stop/pause/restart of a service, and
FAULT_POINTS of ONE service recreated with an override (conftest.service_with_env) - and every
test restores the normal stand in `finally`. Each test checks exact HTTP codes AND the rows in
PostgreSQL AND the objects in S3, then the read-only invariant audit (I1-I7) of the whole stand."""
from __future__ import annotations
import hashlib
import io
import json
import subprocess
import time
import uuid
from pathlib import Path
import pytest
from PIL import Image
import imagegen
from conftest import (attachment_row, audit, compose, events_of, job_rows, jobctl, psql, queue_depth, s3_keys,
s3_size, service_down, service_with_env, wait_healthy, wait_stand)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
FAST_LEASE = {"JOB_LEASE_SECONDS": "6"} # the reaper notices a dead attempt within ~11 s
@pytest.fixture()
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
def wait_for(condition, timeout: float, what: str):
deadline = time.monotonic() + timeout
while True:
value = condition()
if value:
return value
assert time.monotonic() < deadline, what
time.sleep(0.5)
def check_variants(api, actor, att: dict, source: bytes) -> dict[str, bytes]:
"""Download every variant: real format and size decoded with Pillow, size/sha256 = metadata."""
expected = {"thumbnail": ("image/jpeg", "JPEG"), "crop": ("image/jpeg", "JPEG"), "optimized": ("image/webp", "WEBP")}
assert [v["name"] for v in att["variants"]] == list(expected)
src = Image.open(io.BytesIO(source))
w, h = src.size
out = {}
for v in att["variants"]:
got = api.content(actor, att["id"], v["name"])
assert got.headers["content-type"] == expected[v["name"]][0] == v["content_type"]
assert len(got.content) == v["size_bytes"] and hashlib.sha256(got.content).hexdigest() == v["sha256"]
img = Image.open(io.BytesIO(got.content))
assert img.format == expected[v["name"]][1] and img.size == (v["width"], v["height"])
out[v["name"]] = got.content
def fit(box):
if w <= box and h <= box:
return w, h
return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box)
sizes = {v["name"]: (v["width"], v["height"]) for v in att["variants"]}
assert sizes == {"thumbnail": fit(256), "crop": (128, 128), "optimized": fit(1280)}
original = api.content(actor, att["id"])
assert original.content == source # the original is published byte for byte
return out
def published_keys(attachment_id: str) -> list[str]:
return psql(f"SELECT object_key FROM attachment_variants WHERE attachment_id = '{attachment_id}' "
f"UNION ALL SELECT object_key FROM attachments WHERE id = '{attachment_id}' ORDER BY 1").split()
def attempts(attachment_id: str) -> list[dict]:
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', a.attempt, 'outcome', a.outcome, "
"'error_code', a.error_code, 'token', a.token, 'cleaned', a.objects_cleaned_at IS NOT NULL, "
"'keys', a.planned_keys) ORDER BY a.started_at), '[]') FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
f"WHERE j.attachment_id = '{attachment_id}'")
return json.loads(out)
def outbox_of(attachment_id: str) -> list[dict]:
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', o.attempt, 'published', o.published_at IS NOT NULL) "
f"ORDER BY o.id), '[]') FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}'")
return json.loads(out)
def names(events: list[dict]) -> list[str]:
return [e["event"] for e in events]
# ------------------------------------------------------------------ level 3: the happy path
def test_upload_is_202_queued_then_ready_with_contract_variants(api, pair):
alice, bob, chat = pair
data = imagegen.png(900, 600, noise=True)
response = api.upload(alice, chat, data)
assert response.status_code == 202
att = response.json()
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None
ready = api.wait_status(bob, att["id"], "ready")
assert ready["error_code"] is None and ready["sha256"] == hashlib.sha256(data).hexdigest()
check_variants(api, bob, ready, data)
row = attachment_row(att["id"])
assert row["published_version"] == 1 and row["staging_state"] == "deleted" # staging cleaned after ready
assert s3_size(row["staging_key"], "staging") is None
[job] = job_rows(att["id"])
assert job["state"] == "succeeded" and job["attempt"] == 1 and job["kind"] == "upload"
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
def test_queued_attachment_is_attachable_failed_is_not(api, pair):
alice, bob, chat = pair
with service_down("worker"): # stays queued
att = api.upload(alice, chat, SAMPLE).json()
msg = api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
assert msg["attachment_ids"] == [att["id"]]
api.wait_status(alice, att["id"], "ready")
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
bad = api.upload(alice, chat, SAMPLE).json()
failed = api.wait_status(bob, bad["id"], "failed")
assert failed["error_code"] == "invalid_image" and failed["variants"] == []
api.post_message(alice, chat, {"text": "", "attachment_ids": [bad["id"]]}, 409)
api.content(bob, bad["id"], expected=409)
api.content(bob, bad["id"], "thumbnail", expected=409)
def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready(api, pair):
"""The defence scenario: worker stopped -> upload 202 -> content 409, other messages keep working
-> the message with the attachment is created -> API and broker restarted WITH volumes ->
the job is still in the queue -> worker started -> ready, thumbnail downloadable."""
alice, bob, chat = pair
compose("stop", "worker")
try:
response = api.upload(alice, chat, SAMPLE)
att = response.json()
assert response.status_code == 202 and att["status"] == "queued"
for variant in (None, "thumbnail"):
assert api.content(bob, att["id"], variant, expected=409).json()["error"]["code"] == "conflict"
api.send(bob, chat, "текст работает без worker")
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
assert queue_depth() >= 1 # durable message waits in the queue
compose("restart", "api", "broker") # volumes preserved
wait_healthy("api", "broker", "relay")
wait_stand()
assert queue_depth() >= 1, "the accepted job was lost by the broker restart"
assert api.attachment(bob, att["id"])["status"] == "queued"
finally:
compose("start", "worker")
wait_healthy("worker")
ready = api.wait_status(bob, att["id"], "ready")
check_variants(api, bob, ready, SAMPLE)
items, _ = api.history(bob, chat, 50)
assert any(m["id"] == msg["id"] and m["attachment_ids"] == [att["id"]] for m in items)
[job] = job_rows(att["id"])
assert job["attempt"] == 1 # no duplicate work
# ------------------------------------------------------------------ failure windows
def test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox(api, pair):
"""Window "after COMMIT, before publish": the broker is down, the API still answers 202 (it never
talks to the broker), the outbox row stays unpublished. No manual action: when the broker is
back the relay publishes it and the worker finishes the job."""
alice, bob, chat = pair
with service_down("broker"):
att = api.upload(alice, chat, SAMPLE).json()
assert att["status"] == "queued"
time.sleep(3)
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
api.send(alice, chat, "сообщения работают без брокера")
api.wait_status(bob, att["id"], "ready", timeout=90)
assert outbox_of(att["id"]) == [{"attempt": 1, "published": True}]
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
def test_relay_down_job_waits_in_outbox_not_lost(api, pair):
alice, bob, chat = pair
with service_down("relay"):
att = api.upload(alice, chat, SAMPLE).json()
time.sleep(3)
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
assert api.attachment(bob, att["id"])["status"] == "queued"
api.wait_status(bob, att["id"], "ready")
def test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate(api, pair):
"""Window "after publish, before confirm": the relay dies after the broker confirmed the message
but before outbox.published_at is committed. The restarted relay publishes it AGAIN; the
duplicate is a no-op (one claim, one set of variants)."""
alice, bob, chat = pair
with service_with_env("relay", RELAY_FAULT_POINTS="crash_after_publish"):
att = api.upload(alice, chat, SAMPLE).json()
api.wait_status(bob, att["id"], "ready", timeout=90)
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 30, "outbox not marked after restart")
events = events_of(att["id"])
assert names(events).count("fault_injected") == 1 and names(events).count("claimed") == 1
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
assert len(published_keys(att["id"])) == 4 # original + 3 variants, once
def test_duplicate_and_reordered_deliveries_are_noops(api, pair):
alice, bob, chat = pair
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish,reverse_order"):
atts = [api.upload(alice, chat, imagegen.png(50 + i, 40)).json() for i in range(4)]
for att in atts:
api.wait_status(bob, att["id"], "ready")
time.sleep(2) # let the duplicates be consumed
for att in atts:
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
assert len(published_keys(att["id"])) == 4
logs = compose("logs", "--no-log-prefix", "worker").stdout
assert '"noop_finished"' in logs or '"noop_leased"' in logs # duplicates were seen and acked
def test_worker_crash_before_side_effect_lease_expires_and_job_completes(api, pair):
"""The worker dies right after the claim. The redelivered message finds a valid lease (no-op);
the reaper returns the job to queued when the lease expires; attempt 2 publishes."""
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_claim", **FAST_LEASE):
att = api.upload(alice, chat, SAMPLE).json()
api.wait_status(bob, att["id"], "ready", timeout=90)
ev = names(events_of(att["id"]))
assert ev.count("fault_injected") == 1 and "lease_expired" in ev and ev[-1] == "succeeded"
tries = attempts(att["id"])
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
assert tries[0]["keys"] and all(s3_size(k) is None for k in tries[0]["keys"]) # nothing written
def test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates(api, pair):
"""Window "after the S3 write, before the result COMMIT": objects of attempt 1 exist under ITS keys
but are never published; attempt 2 publishes its own; the cleanup deletes attempt 1's objects
after ORPHAN_GRACE_SECONDS (they can never become published: the token is fenced)."""
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_s3_write", ORPHAN_GRACE_SECONDS="2",
CLEANUP_INTERVAL_SECONDS="1", **FAST_LEASE):
att = api.upload(alice, chat, SAMPLE).json()
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
first, second = attempts(att["id"])
assert (first["outcome"], second["outcome"]) == ("abandoned", "succeeded")
assert set(published_keys(att["id"])) == set(second["keys"])
wait_for(lambda: attempts(att["id"])[0]["cleaned"], 30, "orphans of attempt 1 not cleaned")
assert all(s3_size(k) is None for k in first["keys"])
assert all(s3_size(k) is not None for k in second["keys"])
assert s3_keys(f"a/{att['id']}/") == set(second["keys"]) # exactly one set of objects
check_variants(api, bob, ready, SAMPLE)
def test_worker_crash_after_commit_before_ack_redelivery_is_noop(api, pair):
"""Window "after the result COMMIT, before the ack": the message comes back; the job is
succeeded -> ack without any work; the same result stays published."""
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_commit"):
att = api.upload(alice, chat, SAMPLE).json()
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
before = published_keys(att["id"])
wait_healthy("worker")
wait_for(lambda: '"noop_finished"' in compose("logs", "--no-log-prefix", "worker").stdout, 30,
"the redelivered message was not acknowledged as a no-op")
ev = names(events_of(att["id"]))
assert ev == ["created", "claimed", "fault_injected", "succeeded"] or ev == ["created", "claimed", "succeeded", "fault_injected"]
assert published_keys(att["id"]) == before and [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
assert api.attachment(bob, att["id"]) == ready
# ------------------------------------------------------------------ retries and failures
def test_transient_errors_are_retried_with_jittered_backoff(api, pair):
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error=2", JOB_RETRY_BASE_SECONDS="1",
JOB_RETRY_MAX_SECONDS="8"):
att = api.upload(alice, chat, SAMPLE).json()
api.wait_status(bob, att["id"], "ready", timeout=60)
events = events_of(att["id"])
retries = [e for e in events if e["event"] == "retry_scheduled"]
assert [(e["attempt"], e["error_code"]) for e in retries] == [(1, "injected_transient"), (2, "injected_transient")]
for e in retries:
assert 0 <= e["detail"]["delay_s"] <= 1 * 2 ** (e["attempt"] - 1) # full jitter window
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "retry"), (2, "retry"), (3, "succeeded")]
assert [o["attempt"] for o in outbox_of(att["id"])] == [1, 2, 3] # every retry went through the outbox
def test_retry_exhausted_failed_registry_then_replay(api, pair):
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error", JOB_RETRY_BASE_SECONDS="0.2",
JOB_RETRY_MAX_SECONDS="0.5"):
att = api.upload(alice, chat, SAMPLE).json()
failed = api.wait_status(bob, att["id"], "failed", timeout=60)
assert failed["error_code"] == "retry_exhausted" and failed["variants"] == []
[job] = job_rows(att["id"])
assert (job["state"], job["attempt"], job["error_code"]) == ("failed", 5, "retry_exhausted") # JOB_MAX_ATTEMPTS
registry = jobctl("list-failed", "--limit", "200")["failed"]
assert any(r["attachment_id"] == att["id"] and r["error_code"] == "retry_exhausted" for r in registry)
assert attachment_row(att["id"])["staging_state"] == "present" # kept for the replay
# Operator replay: a NEW job (replay_of = the failed one), audit with operator and reason.
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "fault removed")
assert result["result"] == "queued" and result["replay_of"] == job["id"]
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
check_variants(api, bob, ready, SAMPLE)
jobs = job_rows(att["id"])
assert [(j["kind"], j["state"]) for j in jobs] == [("upload", "failed"), ("replay", "succeeded")]
ev = events_of(att["id"])
req = next(e for e in ev if e["event"] == "replay_requested")
assert req["actor"] == "jobctl:alice" and req["detail"]["reason"] == "fault removed"
# "Repeat" never re-runs a finished job: a replay of a ready attachment is refused, nothing created.
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "again", expect_code=3)
assert refused == {"result": "refused", "reason": "already_ready"}
assert len(job_rows(att["id"])) == 2
def test_permanent_error_fails_at_once_without_retries(api, pair):
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
att = api.upload(alice, chat, SAMPLE).json()
failed = api.wait_status(bob, att["id"], "failed")
assert failed["error_code"] == "invalid_image"
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "failed")]
def test_memory_and_time_limits_of_the_worker(api, pair):
"""40 Mpx PNG (valid, accepted by the API) with IMAGE_MAX_MEMORY_MB=200: the child process cannot
allocate the raster -> image_too_large (permanent); the worker itself keeps running.
JOB_TIMEOUT_SECONDS=3 with a 10 s stall -> processing_timeout (transient, retried)."""
alice, bob, chat = pair
with service_with_env("worker", IMAGE_MAX_MEMORY_MB="200"):
att = api.upload(alice, chat, imagegen.png(8000, 5000, compress_level=9)).json()
failed = api.wait_status(bob, att["id"], "failed", timeout=90)
assert failed["error_code"] == "image_too_large"
ok = api.upload(alice, chat, SAMPLE).json() # the same worker still works
api.wait_status(bob, ok["id"], "ready")
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10", JOB_TIMEOUT_SECONDS="3"):
slow = api.upload(alice, chat, SAMPLE).json()
wait_for(lambda: any(a["error_code"] == "processing_timeout" for a in attempts(slow["id"])), 60,
"no processing_timeout")
api.wait_status(bob, slow["id"], "ready", timeout=120) # normal worker finishes it
def test_bad_message_is_dead_lettered_not_processed(api, pair):
"""A message that does not match jobs.schema.json (published here with the relay's own broker
user) is rejected by the worker into the DLQ; nothing is processed."""
before = queue_depth("messenger.attachment-jobs.dlq")
script = ("import asyncio, aio_pika\n"
"async def m():\n"
" url = open('/run/secrets/broker_url').read().strip()\n"
" c = await aio_pika.connect(url)\n"
" ch = await c.channel(publisher_confirms=True)\n"
" ex = await ch.get_exchange('messenger.jobs', ensure=False)\n"
" await ex.publish(aio_pika.Message(b'{\"url\":\"http://evil/x.png\"}'), routing_key='attachment.process.v1')\n"
" await c.close()\n"
"asyncio.run(m())\n")
compose("exec", "-T", "relay", "python", "-c", script)
wait_for(lambda: queue_depth("messenger.attachment-jobs.dlq") == before + 1, 20, "not dead-lettered")
assert '"bad_message"' in compose("logs", "--no-log-prefix", "worker").stdout
# ------------------------------------------------------------------ lease, hung worker, shutdown, concurrency
def worker_containers() -> dict[str, str]:
"""hostname (= lease_owner) -> container id of every running worker."""
ids = compose("ps", "-q", "worker").stdout.split()
out = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True,
text=True, check=True).stdout
return dict(line.split() for line in out.splitlines())
def test_hung_worker_loses_its_lease_and_cannot_publish(api, pair):
"""Two workers. The one processing the job is frozen (docker pause: no heartbeat). Its lease
expires, the reaper re-queues the job, the other worker publishes it. When the frozen worker
resumes, its result is fenced by the token: the published result does not change."""
alice, bob, chat = pair
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST_LEASE):
att = api.upload(alice, chat, SAMPLE).json()
owner = wait_for(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
"AND state = 'processing'"), 30, "job never claimed")
frozen = worker_containers()[owner]
subprocess.run(["docker", "pause", frozen], check=True, capture_output=True)
try:
ready = api.wait_status(bob, att["id"], "ready", timeout=120)
published = published_keys(att["id"])
finally:
subprocess.run(["docker", "unpause", frozen], check=True, capture_output=True)
time.sleep(12) # the resumed attempt finishes its sleep and tries to publish
assert published_keys(att["id"]) == published and api.attachment(bob, att["id"]) == ready
tries = attempts(att["id"])
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
assert "lease_expired" in names(events_of(att["id"]))
check_variants(api, bob, ready, SAMPLE)
def test_graceful_worker_stop_finishes_the_running_attempt(api, pair):
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="slow=5"):
att = api.upload(alice, chat, SAMPLE).json()
wait_for(lambda: api.attachment(bob, att["id"])["status"] == "processing", 30, "never processing")
started = time.monotonic()
compose("stop", "worker") # SIGTERM: drain, not kill
assert time.monotonic() - started < 30
assert attachment_row(att["id"])["status"] == "ready" # committed before the exit
compose("start", "worker")
wait_healthy("worker")
assert [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
def test_concurrency_is_bounded_by_prefetch(api, pair):
"""WORKER_CONCURRENCY=2: never more than 2 jobs processing and 2 unacked messages at once."""
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="slow=3"):
atts = [api.upload(alice, chat, imagegen.png(30 + i, 30)).json() for i in range(6)]
peak = 0
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
states = [api.attachment(bob, a["id"])["status"] for a in atts]
peak = max(peak, states.count("processing"))
unacked = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name",
"messages_unacknowledged").stdout
for line in unacked.splitlines():
if line.startswith("messenger.attachment-jobs\t"):
assert int(line.split("\t")[1]) <= 2, line
if all(s == "ready" for s in states):
break
time.sleep(0.3)
assert all(api.attachment(bob, a["id"])["status"] == "ready" for a in atts)
assert peak == 2
# ------------------------------------------------------------------ cleanup never deletes accepted work
def test_queued_staging_is_never_deleted_even_after_retention(api, pair):
"""Reviewer's case in another PR: the worker was down longer than the retention and the cleanup
deleted the original of a job that was still queued. Here: the job waits queued (relay down)
while the cleanup runs every second with retention 0 - the staging copy stays, and the
database itself refuses marking it (CHECK attachments_active_has_source)."""
alice, bob, chat = pair
with service_with_env("worker", STAGING_RETENTION_SECONDS="0", CLEANUP_INTERVAL_SECONDS="1"):
with service_down("relay"):
att = api.upload(alice, chat, SAMPLE).json()
time.sleep(5) # several cleanup passes
row = attachment_row(att["id"])
assert row["status"] == "queued" and row["staging_state"] == "present"
assert s3_size(row["staging_key"], "staging") == len(SAMPLE)
refused = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
f"UPDATE attachments SET staging_state = 'deleting' WHERE id = '{att['id']}'",
check=False)
assert refused.returncode != 0 and "attachments_active_has_source" in refused.stderr
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
check_variants(api, bob, ready, SAMPLE)
def test_retention_deletes_failed_staging_then_replay_is_refused(api, pair):
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error", STAGING_RETENTION_SECONDS="1",
CLEANUP_INTERVAL_SECONDS="1"):
att = api.upload(alice, chat, SAMPLE).json()
api.wait_status(bob, att["id"], "failed")
wait_for(lambda: attachment_row(att["id"])["staging_state"] == "deleted", 30, "staging not cleaned")
assert s3_size(attachment_row(att["id"])["staging_key"], "staging") is None
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "too late", expect_code=3)
assert refused == {"result": "refused", "reason": "source_missing"}
assert api.attachment(bob, att["id"])["status"] == "failed" and len(job_rows(att["id"])) == 1
def test_cleanup_racing_a_replay_never_deletes_the_replayed_source(api, pair):
"""Reviewer's interleaving in another PR: the cleanup READ its candidates (an expired failed
attachment), a replay committed queued, the cleanup went on and deleted the staging copy.
Here the cleanup pauses exactly there (pause_cleanup); the replay commits during the pause;
the cleanup then re-checks under the row lock and skips the row. The replay finishes ready."""
alice, bob, chat = pair
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
att = api.upload(alice, chat, SAMPLE).json()
api.wait_status(bob, att["id"], "failed")
with service_with_env("worker", WORKER_FAULT_POINTS="pause_cleanup=8", STAGING_RETENTION_SECONDS="1",
CLEANUP_INTERVAL_SECONDS="1"):
wait_for(lambda: '"cleanup_paused"' in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout
and att["id"] in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout,
30, "cleanup did not pick the failed attachment")
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "race test")
assert result["result"] == "queued"
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
check_variants(api, bob, ready, SAMPLE)
assert attachment_row(att["id"])["staging_state"] == "deleted" # deleted only AFTER ready
def test_reprocess_of_a_not_ready_upload_is_refused(api, pair):
"""Reviewer's case in another PR: worker stopped -> upload 202 -> reprocess with pipeline 2 ->
the attachment stayed queued forever. Here the command is refused and v1 completes."""
alice, bob, chat = pair
with service_down("worker"):
att = api.upload(alice, chat, SAMPLE).json()
refused = jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "alice", "--reason", "x",
expect_code=3)
assert refused == {"result": "refused", "reason": "not_ready:queued"}
assert len(job_rows(att["id"])) == 1
api.wait_status(bob, att["id"], "ready")
assert attachment_row(att["id"])["published_version"] == 1
def test_zz_invariants_hold_after_all_scenarios():
report = audit("--verify-checksums")
assert report["objects"]["published"] > 0