lab5 уровень 4: jobctl (реестр failed, replay, reprocess), тесты окон отказа и demo

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

jobctl: replay только failed с живым источником (новое задание replay_of + аудит),
reprocess только ready и только более новой версии; решения под той же блокировкой
строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с
jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший
worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий.
This commit is contained in:
Alexey Bodanyuk
2026-10-31 16:48:26 +03:00
parent 1d37fd0dab
commit 962adbe03b
4 changed files with 865 additions and 4 deletions
+220
View File
@@ -0,0 +1,220 @@
"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`).
list-failed [--limit N] failed registry (view failed_jobs), newest first
show <attachment_id> attachment state, its jobs, attempts and audit events
stats jobs by state, outbox backlog
replay <attachment_id> --operator NAME --reason TEXT
failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt
budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source
still exists (staging copy present or original published). "Repeat" never means
"run a finished job again": a ready attachment is refused (already_ready).
reprocess <attachment_id> --pipeline N --operator NAME --reason TEXT
Track 2: build a NEWER pipeline version of a READY attachment from its published original.
Refused unless ready, N known, N > published version and no job of version >= N active.
A not-yet-ready upload is refused: its only source is the staging copy that belongs to the
running job, and its result would race with the first publication.
Every decision is taken under the attachments row lock, the same lock the cleanup takes before it
marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the
row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing).
Output: one JSON document on stdout, without secrets or file names.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import re
import sys
import uuid
from . import jobs
from .config import ConfigError, Settings
from .db import Database
from .errors import Unavailable
from .pipeline import PIPELINES
OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
EXIT_REFUSED = 3
class Refused(Exception):
pass
def operator(value: str) -> str:
if not OPERATOR_RE.fullmatch(value):
raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}")
return value
def reason(value: str) -> str:
value = value.strip()
if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value):
raise argparse.ArgumentTypeError("reason: 1-200 printable characters")
return value
def attachment_id(value: str) -> str:
try:
return str(uuid.UUID(value))
except ValueError:
raise argparse.ArgumentTypeError("not a UUID") from None
async def lock_attachment(conn, attachment: str):
cur = await conn.execute(
"SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments "
"WHERE id = %s FOR UPDATE", (attachment,))
row = await cur.fetchone()
if row is None or row[0] == "pending":
raise Refused("not_found")
return row
async def replay(conn, args, settings: Settings) -> dict:
status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id)
if status == "ready":
raise Refused("already_ready")
if status != "failed":
raise Refused("not_failed:" + status)
if not has_original and staging_state != "present":
raise Refused("source_missing") # staging removed by the retention cleanup
cur = await conn.execute(
"SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' "
"ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,))
last = await cur.fetchone()
if last is None:
raise Refused("no_failed_job")
cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
(args.attachment_id,))
if await cur.fetchone():
raise Refused("active_job")
await conn.execute(
"UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() "
"WHERE id = %s", (args.attachment_id,))
actor = "jobctl:" + args.operator
await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason)
job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor,
replay_of=last[0], reason=args.reason)
return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]}
async def reprocess(conn, args, settings: Settings) -> dict:
status, _, _, published = await lock_attachment(conn, args.attachment_id)
if args.pipeline not in PIPELINES:
raise Refused("unknown_pipeline")
if status != "ready":
raise Refused("not_ready:" + status)
if args.pipeline <= published:
raise Refused("not_newer")
cur = await conn.execute(
"SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
(args.attachment_id,))
(active,) = await cur.fetchone()
if active is not None and active >= args.pipeline:
raise Refused("newer_or_same_active")
job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts,
"jobctl:" + args.operator, reason=args.reason)
return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline,
"published_version": published}
async def list_failed(conn, args, settings: Settings) -> dict:
cur = await conn.execute(
"SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, "
"attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,))
cols = [d.name for d in cur.description]
return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]}
async def show(conn, args, settings: Settings) -> dict:
cur = await conn.execute(
"SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, "
"updated_at FROM attachments WHERE id = %s", (args.attachment_id,))
row = await cur.fetchone()
if row is None:
raise Refused("not_found")
attachment = dict(zip([d.name for d in cur.description], row))
async def rows(sql: str) -> list[dict]:
c = await conn.execute(sql, (args.attachment_id,))
names = [d.name for d in c.description]
return [dict(zip(names, r)) for r in await c.fetchall()]
return {
"attachment": attachment,
"jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, "
"created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"),
"attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, "
"a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a "
"JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"),
"variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, "
"encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s "
"ORDER BY pipeline_version, name"),
"events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail "
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s "
"ORDER BY e.id"),
}
async def stats(conn, args, settings: Settings) -> dict:
cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state")
by_state = {r[0]: r[1] for r in await cur.fetchall()}
cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) "
"FROM outbox WHERE published_at IS NULL")
pending, due = await cur.fetchone()
return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due}
COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats}
def arguments(argv: list[str] | None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess")
sub = p.add_subparsers(dest="command", required=True)
for name in ("replay", "reprocess"):
s = sub.add_parser(name)
s.add_argument("attachment_id", type=attachment_id)
if name == "reprocess":
s.add_argument("--pipeline", type=int, required=True)
s.add_argument("--operator", type=operator, required=True)
s.add_argument("--reason", type=reason, required=True)
s = sub.add_parser("list-failed")
s.add_argument("--limit", type=int, default=50)
s = sub.add_parser("show")
s.add_argument("attachment_id", type=attachment_id)
sub.add_parser("stats")
return p.parse_args(argv)
async def run(args: argparse.Namespace) -> int:
settings = Settings.from_env(s3_credentials=False)
db = Database(settings, name="messenger-jobctl")
await db.open()
try:
writes = args.command in ("replay", "reprocess")
result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10)
print(json.dumps(result, ensure_ascii=False, indent=1, default=str))
return 0 if not writes or result.get("result") == "queued" else 1
except Refused as exc:
print(json.dumps({"result": "refused", "reason": str(exc)}))
return EXIT_REFUSED
except Unavailable:
print(json.dumps({"result": "error", "reason": "database_unavailable"}))
return 1
finally:
await db.close()
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())