forked from mtusi2026/mtusi2026lab5
lab5 уровень 4: jobctl (реестр failed, replay, reprocess), тесты окон отказа и demo
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> jobctl: replay только failed с живым источником (новое задание replay_of + аудит), reprocess только ready и только более новой версии; решения под той же блокировкой строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
|
||||
"""Lab 4 defense scenario (kept in lab 5; upload is now 202 queued, the download waits for ready):
|
||||
upload the fixture, send a message with attachment_ids,
|
||||
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
||||
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
||||
|
||||
@@ -67,6 +68,16 @@ def user(c: Client, label: str) -> dict:
|
||||
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
||||
|
||||
|
||||
def wait_ready(c: Client, actor: dict, attachment_id: str, timeout: float = 60):
|
||||
import time
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{attachment_id}", token=actor["token"])
|
||||
if st != 200 or json.loads(b)["status"] in ("ready", "failed") or time.monotonic() > deadline:
|
||||
return st, h, b
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
@@ -77,7 +88,7 @@ def main() -> int:
|
||||
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
||||
print(f"users: alice, bob (members), eve (outsider)")
|
||||
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
|
||||
print("\n1. upload -> message -> download by the second member")
|
||||
@@ -88,8 +99,8 @@ def main() -> int:
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id} (bob)", st, h, b)
|
||||
st, h, b = wait_ready(c, bob, att["id"])
|
||||
show("GET /attachments/{id} (bob) until ready", st, h, b)
|
||||
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
||||
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 5 defense scenario, steps 2-3 (README): worker stopped -> upload 202 -> content 409 while the
|
||||
rest of the API works -> a message with the attachment is created -> API and broker restarted with
|
||||
their volumes -> worker started -> the same attachment becomes ready, thumbnail downloadable.
|
||||
|
||||
python scripts/demo_pipeline.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints every step with HTTP codes, IDs, the outbox row and the queue depth; never prints tokens or
|
||||
keys. Creates synthetic users. Always starts the worker again at the end.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from demo_attachments import FIXTURE, ROOT, Client, show, user, wait_ready # noqa: E402
|
||||
|
||||
|
||||
def compose(*args: str, check: bool = True) -> str:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
check=check, timeout=300).stdout
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql).strip()
|
||||
|
||||
|
||||
def queue_depth() -> str:
|
||||
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
|
||||
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
|
||||
_, messages, consumers = line.split("\t")
|
||||
return f"messages={messages} consumers={consumers}"
|
||||
|
||||
|
||||
def wait_api(c: Client, timeout: float = 120) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if c.call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("API did not become ready")
|
||||
|
||||
|
||||
def step(text: str) -> None:
|
||||
print(f"\n[{time.strftime('%H:%M:%S')}] {text}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
alice, bob = user(c, "alice"), user(c, "bob")
|
||||
_, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
try:
|
||||
step("1. docker compose stop worker")
|
||||
compose("stop", "worker")
|
||||
print(" " + compose("ps", "--format", "{{.Service}} {{.State}}", "worker").strip())
|
||||
|
||||
step("2. upload while the worker is stopped")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments", st, h, b)
|
||||
att = json.loads(b)
|
||||
print(f" attachment_id = {att['id']} status = {att['status']}")
|
||||
for query in ("", "?variant=thumbnail"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}/content{query}", token=bob["token"])
|
||||
show(f"GET /attachments/{{id}}/content{query}", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=bob["token"],
|
||||
body={"text": "обычное сообщение работает"}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages (text only)", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "фото", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages attachment_ids=[queued id]", st, h, b)
|
||||
message_id = json.loads(b)["id"]
|
||||
time.sleep(2)
|
||||
print(" job + outbox in PostgreSQL: " + psql(
|
||||
"SELECT j.id || ' state=' || j.state || ' attempt=' || j.attempt || ' | outbox published_at=' || "
|
||||
f"coalesce(o.published_at::text, 'NULL') FROM jobs j JOIN outbox o ON o.job_id = j.id "
|
||||
f"WHERE j.attachment_id = '{att['id']}'"))
|
||||
print(" RabbitMQ queue messenger.attachment-jobs: " + queue_depth())
|
||||
|
||||
step("3. docker compose restart api broker (volumes preserved)")
|
||||
compose("restart", "api", "broker")
|
||||
wait_api(c)
|
||||
time.sleep(3)
|
||||
print(" RabbitMQ queue after the restart: " + queue_depth())
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id}", st, h, b)
|
||||
finally:
|
||||
step("4. docker compose start worker")
|
||||
compose("start", "worker")
|
||||
st, h, b = wait_ready(c, bob, att["id"], timeout=90)
|
||||
show("GET /attachments/{id} until ready", st, h, b)
|
||||
ready = json.loads(b)
|
||||
thumb = next(v for v in ready["variants"] if v["name"] == "thumbnail")
|
||||
st, h, data = c.call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=bob["token"])
|
||||
show("GET /content?variant=thumbnail", st, h, data, binary=True)
|
||||
from PIL import Image
|
||||
size = Image.open(io.BytesIO(data)).size
|
||||
ok = (st == 200 and hashlib.sha256(data).hexdigest() == thumb["sha256"] and size == (256, 160)
|
||||
and ready["status"] == "ready")
|
||||
print(f" thumbnail: real JPEG {size[0]}x{size[1]}, sha256 matches metadata: {ok}")
|
||||
st, h, orig = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
print(f" original unchanged byte for byte: {orig == content}")
|
||||
print(f" message {message_id} still references it: "
|
||||
f"{psql(f'SELECT attachment_ids FROM messages WHERE id = {chr(39)}{message_id}{chr(39)}')}")
|
||||
step("5. job history (jobctl show)")
|
||||
shown = json.loads(compose("run", "--rm", "--no-deps", "-T", "jobctl", "show", att["id"]))
|
||||
for e in shown["events"]:
|
||||
print(f" {e['at']} {e['event']:16} attempt={e['attempt']} actor={e['actor']}")
|
||||
return 0 if ok and orig == content else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`).
|
||||
|
||||
list-failed [--limit N] failed registry (view failed_jobs), newest first
|
||||
show <attachment_id> attachment state, its jobs, attempts and audit events
|
||||
stats jobs by state, outbox backlog
|
||||
replay <attachment_id> --operator NAME --reason TEXT
|
||||
failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt
|
||||
budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source
|
||||
still exists (staging copy present or original published). "Repeat" never means
|
||||
"run a finished job again": a ready attachment is refused (already_ready).
|
||||
reprocess <attachment_id> --pipeline N --operator NAME --reason TEXT
|
||||
Track 2: build a NEWER pipeline version of a READY attachment from its published original.
|
||||
Refused unless ready, N known, N > published version and no job of version >= N active.
|
||||
A not-yet-ready upload is refused: its only source is the staging copy that belongs to the
|
||||
running job, and its result would race with the first publication.
|
||||
|
||||
Every decision is taken under the attachments row lock, the same lock the cleanup takes before it
|
||||
marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the
|
||||
row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing).
|
||||
Output: one JSON document on stdout, without secrets or file names.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from . import jobs
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .pipeline import PIPELINES
|
||||
|
||||
OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||||
EXIT_REFUSED = 3
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def operator(value: str) -> str:
|
||||
if not OPERATOR_RE.fullmatch(value):
|
||||
raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}")
|
||||
return value
|
||||
|
||||
|
||||
def reason(value: str) -> str:
|
||||
value = value.strip()
|
||||
if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value):
|
||||
raise argparse.ArgumentTypeError("reason: 1-200 printable characters")
|
||||
return value
|
||||
|
||||
|
||||
def attachment_id(value: str) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("not a UUID") from None
|
||||
|
||||
|
||||
async def lock_attachment(conn, attachment: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments "
|
||||
"WHERE id = %s FOR UPDATE", (attachment,))
|
||||
row = await cur.fetchone()
|
||||
if row is None or row[0] == "pending":
|
||||
raise Refused("not_found")
|
||||
return row
|
||||
|
||||
|
||||
async def replay(conn, args, settings: Settings) -> dict:
|
||||
status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id)
|
||||
if status == "ready":
|
||||
raise Refused("already_ready")
|
||||
if status != "failed":
|
||||
raise Refused("not_failed:" + status)
|
||||
if not has_original and staging_state != "present":
|
||||
raise Refused("source_missing") # staging removed by the retention cleanup
|
||||
cur = await conn.execute(
|
||||
"SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' "
|
||||
"ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,))
|
||||
last = await cur.fetchone()
|
||||
if last is None:
|
||||
raise Refused("no_failed_job")
|
||||
cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
if await cur.fetchone():
|
||||
raise Refused("active_job")
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s", (args.attachment_id,))
|
||||
actor = "jobctl:" + args.operator
|
||||
await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason)
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor,
|
||||
replay_of=last[0], reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]}
|
||||
|
||||
|
||||
async def reprocess(conn, args, settings: Settings) -> dict:
|
||||
status, _, _, published = await lock_attachment(conn, args.attachment_id)
|
||||
if args.pipeline not in PIPELINES:
|
||||
raise Refused("unknown_pipeline")
|
||||
if status != "ready":
|
||||
raise Refused("not_ready:" + status)
|
||||
if args.pipeline <= published:
|
||||
raise Refused("not_newer")
|
||||
cur = await conn.execute(
|
||||
"SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
(active,) = await cur.fetchone()
|
||||
if active is not None and active >= args.pipeline:
|
||||
raise Refused("newer_or_same_active")
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts,
|
||||
"jobctl:" + args.operator, reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline,
|
||||
"published_version": published}
|
||||
|
||||
|
||||
async def list_failed(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, "
|
||||
"attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,))
|
||||
cols = [d.name for d in cur.description]
|
||||
return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]}
|
||||
|
||||
|
||||
async def show(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, "
|
||||
"updated_at FROM attachments WHERE id = %s", (args.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise Refused("not_found")
|
||||
attachment = dict(zip([d.name for d in cur.description], row))
|
||||
|
||||
async def rows(sql: str) -> list[dict]:
|
||||
c = await conn.execute(sql, (args.attachment_id,))
|
||||
names = [d.name for d in c.description]
|
||||
return [dict(zip(names, r)) for r in await c.fetchall()]
|
||||
|
||||
return {
|
||||
"attachment": attachment,
|
||||
"jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, "
|
||||
"created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"),
|
||||
"attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, "
|
||||
"a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a "
|
||||
"JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"),
|
||||
"variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, "
|
||||
"encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s "
|
||||
"ORDER BY pipeline_version, name"),
|
||||
"events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail "
|
||||
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s "
|
||||
"ORDER BY e.id"),
|
||||
}
|
||||
|
||||
|
||||
async def stats(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state")
|
||||
by_state = {r[0]: r[1] for r in await cur.fetchall()}
|
||||
cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) "
|
||||
"FROM outbox WHERE published_at IS NULL")
|
||||
pending, due = await cur.fetchone()
|
||||
return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due}
|
||||
|
||||
|
||||
COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats}
|
||||
|
||||
|
||||
def arguments(argv: list[str] | None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
for name in ("replay", "reprocess"):
|
||||
s = sub.add_parser(name)
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
if name == "reprocess":
|
||||
s.add_argument("--pipeline", type=int, required=True)
|
||||
s.add_argument("--operator", type=operator, required=True)
|
||||
s.add_argument("--reason", type=reason, required=True)
|
||||
s = sub.add_parser("list-failed")
|
||||
s.add_argument("--limit", type=int, default=50)
|
||||
s = sub.add_parser("show")
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
sub.add_parser("stats")
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
settings = Settings.from_env(s3_credentials=False)
|
||||
db = Database(settings, name="messenger-jobctl")
|
||||
await db.open()
|
||||
try:
|
||||
writes = args.command in ("replay", "reprocess")
|
||||
result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=1, default=str))
|
||||
return 0 if not writes or result.get("result") == "queued" else 1
|
||||
except Refused as exc:
|
||||
print(json.dumps({"result": "refused", "reason": str(exc)}))
|
||||
return EXIT_REFUSED
|
||||
except Unavailable:
|
||||
print(json.dumps({"result": "error", "reason": "database_unavailable"}))
|
||||
return 1
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = arguments(argv)
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,500 @@
|
||||
"""Lab 5 pipeline against the running stand: 202 -> queued -> ready, variants by the contract,
|
||||
the failure windows of contracts/attachment-states.md, retries, the failed registry and replay,
|
||||
lease expiry of a hung worker, cleanup that never deletes accepted work.
|
||||
|
||||
Failures are injected with the real mechanisms - docker stop/pause/restart of a service, and
|
||||
FAULT_POINTS of ONE service recreated with an override (conftest.service_with_env) - and every
|
||||
test restores the normal stand in `finally`. Each test checks exact HTTP codes AND the rows in
|
||||
PostgreSQL AND the objects in S3, then the read-only invariant audit (I1-I7) of the whole stand."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import imagegen
|
||||
from conftest import (attachment_row, audit, compose, events_of, job_rows, jobctl, psql, queue_depth, s3_keys,
|
||||
s3_size, service_down, service_with_env, wait_healthy, wait_stand)
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
FAST_LEASE = {"JOB_LEASE_SECONDS": "6"} # the reaper notices a dead attempt within ~11 s
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
def wait_for(condition, timeout: float, what: str):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
value = condition()
|
||||
if value:
|
||||
return value
|
||||
assert time.monotonic() < deadline, what
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def check_variants(api, actor, att: dict, source: bytes) -> dict[str, bytes]:
|
||||
"""Download every variant: real format and size decoded with Pillow, size/sha256 = metadata."""
|
||||
expected = {"thumbnail": ("image/jpeg", "JPEG"), "crop": ("image/jpeg", "JPEG"), "optimized": ("image/webp", "WEBP")}
|
||||
assert [v["name"] for v in att["variants"]] == list(expected)
|
||||
src = Image.open(io.BytesIO(source))
|
||||
w, h = src.size
|
||||
out = {}
|
||||
for v in att["variants"]:
|
||||
got = api.content(actor, att["id"], v["name"])
|
||||
assert got.headers["content-type"] == expected[v["name"]][0] == v["content_type"]
|
||||
assert len(got.content) == v["size_bytes"] and hashlib.sha256(got.content).hexdigest() == v["sha256"]
|
||||
img = Image.open(io.BytesIO(got.content))
|
||||
assert img.format == expected[v["name"]][1] and img.size == (v["width"], v["height"])
|
||||
out[v["name"]] = got.content
|
||||
def fit(box):
|
||||
if w <= box and h <= box:
|
||||
return w, h
|
||||
return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box)
|
||||
sizes = {v["name"]: (v["width"], v["height"]) for v in att["variants"]}
|
||||
assert sizes == {"thumbnail": fit(256), "crop": (128, 128), "optimized": fit(1280)}
|
||||
original = api.content(actor, att["id"])
|
||||
assert original.content == source # the original is published byte for byte
|
||||
return out
|
||||
|
||||
|
||||
def published_keys(attachment_id: str) -> list[str]:
|
||||
return psql(f"SELECT object_key FROM attachment_variants WHERE attachment_id = '{attachment_id}' "
|
||||
f"UNION ALL SELECT object_key FROM attachments WHERE id = '{attachment_id}' ORDER BY 1").split()
|
||||
|
||||
|
||||
def attempts(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', a.attempt, 'outcome', a.outcome, "
|
||||
"'error_code', a.error_code, 'token', a.token, 'cleaned', a.objects_cleaned_at IS NOT NULL, "
|
||||
"'keys', a.planned_keys) ORDER BY a.started_at), '[]') FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
f"WHERE j.attachment_id = '{attachment_id}'")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def outbox_of(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', o.attempt, 'published', o.published_at IS NOT NULL) "
|
||||
f"ORDER BY o.id), '[]') FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}'")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def names(events: list[dict]) -> list[str]:
|
||||
return [e["event"] for e in events]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 3: the happy path
|
||||
def test_upload_is_202_queued_then_ready_with_contract_variants(api, pair):
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png(900, 600, noise=True)
|
||||
response = api.upload(alice, chat, data)
|
||||
assert response.status_code == 202
|
||||
att = response.json()
|
||||
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None
|
||||
ready = api.wait_status(bob, att["id"], "ready")
|
||||
assert ready["error_code"] is None and ready["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
check_variants(api, bob, ready, data)
|
||||
row = attachment_row(att["id"])
|
||||
assert row["published_version"] == 1 and row["staging_state"] == "deleted" # staging cleaned after ready
|
||||
assert s3_size(row["staging_key"], "staging") is None
|
||||
[job] = job_rows(att["id"])
|
||||
assert job["state"] == "succeeded" and job["attempt"] == 1 and job["kind"] == "upload"
|
||||
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
||||
|
||||
|
||||
def test_queued_attachment_is_attachable_failed_is_not(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_down("worker"): # stays queued
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
msg = api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
||||
assert msg["attachment_ids"] == [att["id"]]
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
bad = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, bad["id"], "failed")
|
||||
assert failed["error_code"] == "invalid_image" and failed["variants"] == []
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [bad["id"]]}, 409)
|
||||
api.content(bob, bad["id"], expected=409)
|
||||
api.content(bob, bad["id"], "thumbnail", expected=409)
|
||||
|
||||
|
||||
def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready(api, pair):
|
||||
"""The defence scenario: worker stopped -> upload 202 -> content 409, other messages keep working
|
||||
-> the message with the attachment is created -> API and broker restarted WITH volumes ->
|
||||
the job is still in the queue -> worker started -> ready, thumbnail downloadable."""
|
||||
alice, bob, chat = pair
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
response = api.upload(alice, chat, SAMPLE)
|
||||
att = response.json()
|
||||
assert response.status_code == 202 and att["status"] == "queued"
|
||||
for variant in (None, "thumbnail"):
|
||||
assert api.content(bob, att["id"], variant, expected=409).json()["error"]["code"] == "conflict"
|
||||
api.send(bob, chat, "текст работает без worker")
|
||||
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
|
||||
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
|
||||
assert queue_depth() >= 1 # durable message waits in the queue
|
||||
compose("restart", "api", "broker") # volumes preserved
|
||||
wait_healthy("api", "broker", "relay")
|
||||
wait_stand()
|
||||
assert queue_depth() >= 1, "the accepted job was lost by the broker restart"
|
||||
assert api.attachment(bob, att["id"])["status"] == "queued"
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
wait_healthy("worker")
|
||||
ready = api.wait_status(bob, att["id"], "ready")
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
items, _ = api.history(bob, chat, 50)
|
||||
assert any(m["id"] == msg["id"] and m["attachment_ids"] == [att["id"]] for m in items)
|
||||
[job] = job_rows(att["id"])
|
||||
assert job["attempt"] == 1 # no duplicate work
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ failure windows
|
||||
def test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox(api, pair):
|
||||
"""Window "after COMMIT, before publish": the broker is down, the API still answers 202 (it never
|
||||
talks to the broker), the outbox row stays unpublished. No manual action: when the broker is
|
||||
back the relay publishes it and the worker finishes the job."""
|
||||
alice, bob, chat = pair
|
||||
with service_down("broker"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
assert att["status"] == "queued"
|
||||
time.sleep(3)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
||||
api.send(alice, chat, "сообщения работают без брокера")
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": True}]
|
||||
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
||||
|
||||
|
||||
def test_relay_down_job_waits_in_outbox_not_lost(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_down("relay"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
time.sleep(3)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
||||
assert api.attachment(bob, att["id"])["status"] == "queued"
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
|
||||
|
||||
def test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate(api, pair):
|
||||
"""Window "after publish, before confirm": the relay dies after the broker confirmed the message
|
||||
but before outbox.published_at is committed. The restarted relay publishes it AGAIN; the
|
||||
duplicate is a no-op (one claim, one set of variants)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="crash_after_publish"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 30, "outbox not marked after restart")
|
||||
events = events_of(att["id"])
|
||||
assert names(events).count("fault_injected") == 1 and names(events).count("claimed") == 1
|
||||
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
||||
assert len(published_keys(att["id"])) == 4 # original + 3 variants, once
|
||||
|
||||
|
||||
def test_duplicate_and_reordered_deliveries_are_noops(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish,reverse_order"):
|
||||
atts = [api.upload(alice, chat, imagegen.png(50 + i, 40)).json() for i in range(4)]
|
||||
for att in atts:
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
time.sleep(2) # let the duplicates be consumed
|
||||
for att in atts:
|
||||
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
||||
assert len(published_keys(att["id"])) == 4
|
||||
logs = compose("logs", "--no-log-prefix", "worker").stdout
|
||||
assert '"noop_finished"' in logs or '"noop_leased"' in logs # duplicates were seen and acked
|
||||
|
||||
|
||||
def test_worker_crash_before_side_effect_lease_expires_and_job_completes(api, pair):
|
||||
"""The worker dies right after the claim. The redelivered message finds a valid lease (no-op);
|
||||
the reaper returns the job to queued when the lease expires; attempt 2 publishes."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_claim", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
ev = names(events_of(att["id"]))
|
||||
assert ev.count("fault_injected") == 1 and "lease_expired" in ev and ev[-1] == "succeeded"
|
||||
tries = attempts(att["id"])
|
||||
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
||||
assert tries[0]["keys"] and all(s3_size(k) is None for k in tries[0]["keys"]) # nothing written
|
||||
|
||||
|
||||
def test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates(api, pair):
|
||||
"""Window "after the S3 write, before the result COMMIT": objects of attempt 1 exist under ITS keys
|
||||
but are never published; attempt 2 publishes its own; the cleanup deletes attempt 1's objects
|
||||
after ORPHAN_GRACE_SECONDS (they can never become published: the token is fenced)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_s3_write", ORPHAN_GRACE_SECONDS="2",
|
||||
CLEANUP_INTERVAL_SECONDS="1", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
first, second = attempts(att["id"])
|
||||
assert (first["outcome"], second["outcome"]) == ("abandoned", "succeeded")
|
||||
assert set(published_keys(att["id"])) == set(second["keys"])
|
||||
wait_for(lambda: attempts(att["id"])[0]["cleaned"], 30, "orphans of attempt 1 not cleaned")
|
||||
assert all(s3_size(k) is None for k in first["keys"])
|
||||
assert all(s3_size(k) is not None for k in second["keys"])
|
||||
assert s3_keys(f"a/{att['id']}/") == set(second["keys"]) # exactly one set of objects
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_worker_crash_after_commit_before_ack_redelivery_is_noop(api, pair):
|
||||
"""Window "after the result COMMIT, before the ack": the message comes back; the job is
|
||||
succeeded -> ack without any work; the same result stays published."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_commit"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
before = published_keys(att["id"])
|
||||
wait_healthy("worker")
|
||||
wait_for(lambda: '"noop_finished"' in compose("logs", "--no-log-prefix", "worker").stdout, 30,
|
||||
"the redelivered message was not acknowledged as a no-op")
|
||||
ev = names(events_of(att["id"]))
|
||||
assert ev == ["created", "claimed", "fault_injected", "succeeded"] or ev == ["created", "claimed", "succeeded", "fault_injected"]
|
||||
assert published_keys(att["id"]) == before and [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
||||
assert api.attachment(bob, att["id"]) == ready
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ retries and failures
|
||||
def test_transient_errors_are_retried_with_jittered_backoff(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error=2", JOB_RETRY_BASE_SECONDS="1",
|
||||
JOB_RETRY_MAX_SECONDS="8"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
events = events_of(att["id"])
|
||||
retries = [e for e in events if e["event"] == "retry_scheduled"]
|
||||
assert [(e["attempt"], e["error_code"]) for e in retries] == [(1, "injected_transient"), (2, "injected_transient")]
|
||||
for e in retries:
|
||||
assert 0 <= e["detail"]["delay_s"] <= 1 * 2 ** (e["attempt"] - 1) # full jitter window
|
||||
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "retry"), (2, "retry"), (3, "succeeded")]
|
||||
assert [o["attempt"] for o in outbox_of(att["id"])] == [1, 2, 3] # every retry went through the outbox
|
||||
|
||||
|
||||
def test_retry_exhausted_failed_registry_then_replay(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error", JOB_RETRY_BASE_SECONDS="0.2",
|
||||
JOB_RETRY_MAX_SECONDS="0.5"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed", timeout=60)
|
||||
assert failed["error_code"] == "retry_exhausted" and failed["variants"] == []
|
||||
[job] = job_rows(att["id"])
|
||||
assert (job["state"], job["attempt"], job["error_code"]) == ("failed", 5, "retry_exhausted") # JOB_MAX_ATTEMPTS
|
||||
registry = jobctl("list-failed", "--limit", "200")["failed"]
|
||||
assert any(r["attachment_id"] == att["id"] and r["error_code"] == "retry_exhausted" for r in registry)
|
||||
assert attachment_row(att["id"])["staging_state"] == "present" # kept for the replay
|
||||
# Operator replay: a NEW job (replay_of = the failed one), audit with operator and reason.
|
||||
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "fault removed")
|
||||
assert result["result"] == "queued" and result["replay_of"] == job["id"]
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
jobs = job_rows(att["id"])
|
||||
assert [(j["kind"], j["state"]) for j in jobs] == [("upload", "failed"), ("replay", "succeeded")]
|
||||
ev = events_of(att["id"])
|
||||
req = next(e for e in ev if e["event"] == "replay_requested")
|
||||
assert req["actor"] == "jobctl:alice" and req["detail"]["reason"] == "fault removed"
|
||||
# "Repeat" never re-runs a finished job: a replay of a ready attachment is refused, nothing created.
|
||||
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "again", expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "already_ready"}
|
||||
assert len(job_rows(att["id"])) == 2
|
||||
|
||||
|
||||
def test_permanent_error_fails_at_once_without_retries(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed")
|
||||
assert failed["error_code"] == "invalid_image"
|
||||
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "failed")]
|
||||
|
||||
|
||||
def test_memory_and_time_limits_of_the_worker(api, pair):
|
||||
"""40 Mpx PNG (valid, accepted by the API) with IMAGE_MAX_MEMORY_MB=200: the child process cannot
|
||||
allocate the raster -> image_too_large (permanent); the worker itself keeps running.
|
||||
JOB_TIMEOUT_SECONDS=3 with a 10 s stall -> processing_timeout (transient, retried)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", IMAGE_MAX_MEMORY_MB="200"):
|
||||
att = api.upload(alice, chat, imagegen.png(8000, 5000, compress_level=9)).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed", timeout=90)
|
||||
assert failed["error_code"] == "image_too_large"
|
||||
ok = api.upload(alice, chat, SAMPLE).json() # the same worker still works
|
||||
api.wait_status(bob, ok["id"], "ready")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10", JOB_TIMEOUT_SECONDS="3"):
|
||||
slow = api.upload(alice, chat, SAMPLE).json()
|
||||
wait_for(lambda: any(a["error_code"] == "processing_timeout" for a in attempts(slow["id"])), 60,
|
||||
"no processing_timeout")
|
||||
api.wait_status(bob, slow["id"], "ready", timeout=120) # normal worker finishes it
|
||||
|
||||
|
||||
def test_bad_message_is_dead_lettered_not_processed(api, pair):
|
||||
"""A message that does not match jobs.schema.json (published here with the relay's own broker
|
||||
user) is rejected by the worker into the DLQ; nothing is processed."""
|
||||
before = queue_depth("messenger.attachment-jobs.dlq")
|
||||
script = ("import asyncio, aio_pika\n"
|
||||
"async def m():\n"
|
||||
" url = open('/run/secrets/broker_url').read().strip()\n"
|
||||
" c = await aio_pika.connect(url)\n"
|
||||
" ch = await c.channel(publisher_confirms=True)\n"
|
||||
" ex = await ch.get_exchange('messenger.jobs', ensure=False)\n"
|
||||
" await ex.publish(aio_pika.Message(b'{\"url\":\"http://evil/x.png\"}'), routing_key='attachment.process.v1')\n"
|
||||
" await c.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
compose("exec", "-T", "relay", "python", "-c", script)
|
||||
wait_for(lambda: queue_depth("messenger.attachment-jobs.dlq") == before + 1, 20, "not dead-lettered")
|
||||
assert '"bad_message"' in compose("logs", "--no-log-prefix", "worker").stdout
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ lease, hung worker, shutdown, concurrency
|
||||
def worker_containers() -> dict[str, str]:
|
||||
"""hostname (= lease_owner) -> container id of every running worker."""
|
||||
ids = compose("ps", "-q", "worker").stdout.split()
|
||||
out = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True,
|
||||
text=True, check=True).stdout
|
||||
return dict(line.split() for line in out.splitlines())
|
||||
|
||||
|
||||
def test_hung_worker_loses_its_lease_and_cannot_publish(api, pair):
|
||||
"""Two workers. The one processing the job is frozen (docker pause: no heartbeat). Its lease
|
||||
expires, the reaper re-queues the job, the other worker publishes it. When the frozen worker
|
||||
resumes, its result is fenced by the token: the published result does not change."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
owner = wait_for(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
|
||||
"AND state = 'processing'"), 30, "job never claimed")
|
||||
frozen = worker_containers()[owner]
|
||||
subprocess.run(["docker", "pause", frozen], check=True, capture_output=True)
|
||||
try:
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=120)
|
||||
published = published_keys(att["id"])
|
||||
finally:
|
||||
subprocess.run(["docker", "unpause", frozen], check=True, capture_output=True)
|
||||
time.sleep(12) # the resumed attempt finishes its sleep and tries to publish
|
||||
assert published_keys(att["id"]) == published and api.attachment(bob, att["id"]) == ready
|
||||
tries = attempts(att["id"])
|
||||
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
||||
assert "lease_expired" in names(events_of(att["id"]))
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_graceful_worker_stop_finishes_the_running_attempt(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=5"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
wait_for(lambda: api.attachment(bob, att["id"])["status"] == "processing", 30, "never processing")
|
||||
started = time.monotonic()
|
||||
compose("stop", "worker") # SIGTERM: drain, not kill
|
||||
assert time.monotonic() - started < 30
|
||||
assert attachment_row(att["id"])["status"] == "ready" # committed before the exit
|
||||
compose("start", "worker")
|
||||
wait_healthy("worker")
|
||||
assert [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
||||
|
||||
|
||||
def test_concurrency_is_bounded_by_prefetch(api, pair):
|
||||
"""WORKER_CONCURRENCY=2: never more than 2 jobs processing and 2 unacked messages at once."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=3"):
|
||||
atts = [api.upload(alice, chat, imagegen.png(30 + i, 30)).json() for i in range(6)]
|
||||
peak = 0
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
states = [api.attachment(bob, a["id"])["status"] for a in atts]
|
||||
peak = max(peak, states.count("processing"))
|
||||
unacked = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name",
|
||||
"messages_unacknowledged").stdout
|
||||
for line in unacked.splitlines():
|
||||
if line.startswith("messenger.attachment-jobs\t"):
|
||||
assert int(line.split("\t")[1]) <= 2, line
|
||||
if all(s == "ready" for s in states):
|
||||
break
|
||||
time.sleep(0.3)
|
||||
assert all(api.attachment(bob, a["id"])["status"] == "ready" for a in atts)
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cleanup never deletes accepted work
|
||||
def test_queued_staging_is_never_deleted_even_after_retention(api, pair):
|
||||
"""Reviewer's case in another PR: the worker was down longer than the retention and the cleanup
|
||||
deleted the original of a job that was still queued. Here: the job waits queued (relay down)
|
||||
while the cleanup runs every second with retention 0 - the staging copy stays, and the
|
||||
database itself refuses marking it (CHECK attachments_active_has_source)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", STAGING_RETENTION_SECONDS="0", CLEANUP_INTERVAL_SECONDS="1"):
|
||||
with service_down("relay"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
time.sleep(5) # several cleanup passes
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "queued" and row["staging_state"] == "present"
|
||||
assert s3_size(row["staging_key"], "staging") == len(SAMPLE)
|
||||
refused = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
||||
f"UPDATE attachments SET staging_state = 'deleting' WHERE id = '{att['id']}'",
|
||||
check=False)
|
||||
assert refused.returncode != 0 and "attachments_active_has_source" in refused.stderr
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_retention_deletes_failed_staging_then_replay_is_refused(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error", STAGING_RETENTION_SECONDS="1",
|
||||
CLEANUP_INTERVAL_SECONDS="1"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "failed")
|
||||
wait_for(lambda: attachment_row(att["id"])["staging_state"] == "deleted", 30, "staging not cleaned")
|
||||
assert s3_size(attachment_row(att["id"])["staging_key"], "staging") is None
|
||||
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "too late", expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "source_missing"}
|
||||
assert api.attachment(bob, att["id"])["status"] == "failed" and len(job_rows(att["id"])) == 1
|
||||
|
||||
|
||||
def test_cleanup_racing_a_replay_never_deletes_the_replayed_source(api, pair):
|
||||
"""Reviewer's interleaving in another PR: the cleanup READ its candidates (an expired failed
|
||||
attachment), a replay committed queued, the cleanup went on and deleted the staging copy.
|
||||
Here the cleanup pauses exactly there (pause_cleanup); the replay commits during the pause;
|
||||
the cleanup then re-checks under the row lock and skips the row. The replay finishes ready."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "failed")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="pause_cleanup=8", STAGING_RETENTION_SECONDS="1",
|
||||
CLEANUP_INTERVAL_SECONDS="1"):
|
||||
wait_for(lambda: '"cleanup_paused"' in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout
|
||||
and att["id"] in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout,
|
||||
30, "cleanup did not pick the failed attachment")
|
||||
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "race test")
|
||||
assert result["result"] == "queued"
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
assert attachment_row(att["id"])["staging_state"] == "deleted" # deleted only AFTER ready
|
||||
|
||||
|
||||
def test_reprocess_of_a_not_ready_upload_is_refused(api, pair):
|
||||
"""Reviewer's case in another PR: worker stopped -> upload 202 -> reprocess with pipeline 2 ->
|
||||
the attachment stayed queued forever. Here the command is refused and v1 completes."""
|
||||
alice, bob, chat = pair
|
||||
with service_down("worker"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
refused = jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "alice", "--reason", "x",
|
||||
expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "not_ready:queued"}
|
||||
assert len(job_rows(att["id"])) == 1
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
assert attachment_row(att["id"])["published_version"] == 1
|
||||
|
||||
|
||||
def test_zz_invariants_hold_after_all_scenarios():
|
||||
report = audit("--verify-checksums")
|
||||
assert report["objects"]["published"] > 0
|
||||
Reference in New Issue
Block a user