forked from mtusi2026/mtusi2026lab5
lab5 уровень 4: jobctl (реестр failed, replay, reprocess), тесты окон отказа и demo
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> jobctl: replay только failed с живым источником (новое задание replay_of + аудит), reprocess только ready и только более новой версии; решения под той же блокировкой строки, что и уборка. test_pipeline_flow: все окна отказа attachment-states.md, retry с jitter, retry_exhausted -> replay, постоянные ошибки, лимиты памяти/времени, DLQ, зависший worker (docker pause), graceful stop, prefetch, уборка vs replay, кейсы из рецензий.
This commit is contained in:
@@ -0,0 +1,220 @@
|
||||
"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`).
|
||||
|
||||
list-failed [--limit N] failed registry (view failed_jobs), newest first
|
||||
show <attachment_id> attachment state, its jobs, attempts and audit events
|
||||
stats jobs by state, outbox backlog
|
||||
replay <attachment_id> --operator NAME --reason TEXT
|
||||
failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt
|
||||
budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source
|
||||
still exists (staging copy present or original published). "Repeat" never means
|
||||
"run a finished job again": a ready attachment is refused (already_ready).
|
||||
reprocess <attachment_id> --pipeline N --operator NAME --reason TEXT
|
||||
Track 2: build a NEWER pipeline version of a READY attachment from its published original.
|
||||
Refused unless ready, N known, N > published version and no job of version >= N active.
|
||||
A not-yet-ready upload is refused: its only source is the staging copy that belongs to the
|
||||
running job, and its result would race with the first publication.
|
||||
|
||||
Every decision is taken under the attachments row lock, the same lock the cleanup takes before it
|
||||
marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the
|
||||
row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing).
|
||||
Output: one JSON document on stdout, without secrets or file names.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from . import jobs
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .pipeline import PIPELINES
|
||||
|
||||
OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||||
EXIT_REFUSED = 3
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def operator(value: str) -> str:
|
||||
if not OPERATOR_RE.fullmatch(value):
|
||||
raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}")
|
||||
return value
|
||||
|
||||
|
||||
def reason(value: str) -> str:
|
||||
value = value.strip()
|
||||
if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value):
|
||||
raise argparse.ArgumentTypeError("reason: 1-200 printable characters")
|
||||
return value
|
||||
|
||||
|
||||
def attachment_id(value: str) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("not a UUID") from None
|
||||
|
||||
|
||||
async def lock_attachment(conn, attachment: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments "
|
||||
"WHERE id = %s FOR UPDATE", (attachment,))
|
||||
row = await cur.fetchone()
|
||||
if row is None or row[0] == "pending":
|
||||
raise Refused("not_found")
|
||||
return row
|
||||
|
||||
|
||||
async def replay(conn, args, settings: Settings) -> dict:
|
||||
status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id)
|
||||
if status == "ready":
|
||||
raise Refused("already_ready")
|
||||
if status != "failed":
|
||||
raise Refused("not_failed:" + status)
|
||||
if not has_original and staging_state != "present":
|
||||
raise Refused("source_missing") # staging removed by the retention cleanup
|
||||
cur = await conn.execute(
|
||||
"SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' "
|
||||
"ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,))
|
||||
last = await cur.fetchone()
|
||||
if last is None:
|
||||
raise Refused("no_failed_job")
|
||||
cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
if await cur.fetchone():
|
||||
raise Refused("active_job")
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s", (args.attachment_id,))
|
||||
actor = "jobctl:" + args.operator
|
||||
await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason)
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor,
|
||||
replay_of=last[0], reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]}
|
||||
|
||||
|
||||
async def reprocess(conn, args, settings: Settings) -> dict:
|
||||
status, _, _, published = await lock_attachment(conn, args.attachment_id)
|
||||
if args.pipeline not in PIPELINES:
|
||||
raise Refused("unknown_pipeline")
|
||||
if status != "ready":
|
||||
raise Refused("not_ready:" + status)
|
||||
if args.pipeline <= published:
|
||||
raise Refused("not_newer")
|
||||
cur = await conn.execute(
|
||||
"SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
(active,) = await cur.fetchone()
|
||||
if active is not None and active >= args.pipeline:
|
||||
raise Refused("newer_or_same_active")
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts,
|
||||
"jobctl:" + args.operator, reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline,
|
||||
"published_version": published}
|
||||
|
||||
|
||||
async def list_failed(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, "
|
||||
"attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,))
|
||||
cols = [d.name for d in cur.description]
|
||||
return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]}
|
||||
|
||||
|
||||
async def show(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, "
|
||||
"updated_at FROM attachments WHERE id = %s", (args.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise Refused("not_found")
|
||||
attachment = dict(zip([d.name for d in cur.description], row))
|
||||
|
||||
async def rows(sql: str) -> list[dict]:
|
||||
c = await conn.execute(sql, (args.attachment_id,))
|
||||
names = [d.name for d in c.description]
|
||||
return [dict(zip(names, r)) for r in await c.fetchall()]
|
||||
|
||||
return {
|
||||
"attachment": attachment,
|
||||
"jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, "
|
||||
"created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"),
|
||||
"attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, "
|
||||
"a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a "
|
||||
"JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"),
|
||||
"variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, "
|
||||
"encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s "
|
||||
"ORDER BY pipeline_version, name"),
|
||||
"events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail "
|
||||
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s "
|
||||
"ORDER BY e.id"),
|
||||
}
|
||||
|
||||
|
||||
async def stats(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state")
|
||||
by_state = {r[0]: r[1] for r in await cur.fetchall()}
|
||||
cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) "
|
||||
"FROM outbox WHERE published_at IS NULL")
|
||||
pending, due = await cur.fetchone()
|
||||
return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due}
|
||||
|
||||
|
||||
COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats}
|
||||
|
||||
|
||||
def arguments(argv: list[str] | None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
for name in ("replay", "reprocess"):
|
||||
s = sub.add_parser(name)
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
if name == "reprocess":
|
||||
s.add_argument("--pipeline", type=int, required=True)
|
||||
s.add_argument("--operator", type=operator, required=True)
|
||||
s.add_argument("--reason", type=reason, required=True)
|
||||
s = sub.add_parser("list-failed")
|
||||
s.add_argument("--limit", type=int, default=50)
|
||||
s = sub.add_parser("show")
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
sub.add_parser("stats")
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
settings = Settings.from_env(s3_credentials=False)
|
||||
db = Database(settings, name="messenger-jobctl")
|
||||
await db.open()
|
||||
try:
|
||||
writes = args.command in ("replay", "reprocess")
|
||||
result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=1, default=str))
|
||||
return 0 if not writes or result.get("result") == "queued" else 1
|
||||
except Refused as exc:
|
||||
print(json.dumps({"result": "refused", "reason": str(exc)}))
|
||||
return EXIT_REFUSED
|
||||
except Unavailable:
|
||||
print(json.dumps({"result": "error", "reason": "database_unavailable"}))
|
||||
return 1
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = arguments(argv)
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user