Files
mtusi2026lab5/app/ops.py

254 lines
12 KiB
Python

"""Operator CLI (runs inside the worker image, never exposed over HTTP).
python -m app.ops failed [--limit N] registry of failed jobs (the "DLQ" of the DB)
python -m app.ops replay <attachment_id> --actor NAME explicit failed -> queued, audited
python -m app.ops reprocess <attachment_id> --pipeline-version N new job for a newer pipeline version
python -m app.ops inspect <attachment_id> JSON: attachment, jobs, variants, events, S3 keys
python -m app.ops stats queue depth, DLQ, outbox backlog, job states
python -m app.ops fault arm <point> <action> [--attachment ID] [--count N]
python -m app.ops fault clear
python -m app.ops republish <attachment_id> [--attempt N] [--pipeline-version N]
raw publish bypassing the outbox: duplicate / stale / reordered message for fault tests
python -m app.ops dlq-peek show (and requeue nothing from) the broker DLQ
"""
from __future__ import annotations
import argparse
import asyncio
import json
import sys
import uuid
import asyncpg
from app import config, faults, jobs
from app.broker import Broker, job_payload
from app.objects import FINAL_PREFIX, ObjectStore
ACTOR_PREFIX = "operator:"
def _ts(v):
return v.isoformat() if v is not None else None
async def connect():
return await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
async def cmd_failed(args) -> int:
conn = await connect()
rows = await conn.fetch(
"""
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, j.last_error,
j.replays, j.updated_at, a.staging_deleted_at IS NULL AS staging_present
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT $1
""", args.limit)
print(json.dumps([{k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v)
for k, v in dict(r).items()} for r in rows], ensure_ascii=False, indent=1))
await conn.close()
return 0
async def cmd_replay(args) -> int:
"""failed -> queued. Refuses anything that is not failed: a ready job is never executed again."""
if not args.actor:
print("--actor is required for the audit log", file=sys.stderr)
return 2
conn = await connect()
try:
async with conn.transaction():
job = await conn.fetchrow(
"""
SELECT j.*, a.status AS a_status, a.staging_deleted_at FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a
""", uuid.UUID(args.attachment_id))
if job is None:
print("no job for this attachment", file=sys.stderr)
return 1
if job["status"] != "failed":
await jobs.event(conn, job["job_id"], job["attachment_id"], "replay_refused", ACTOR_PREFIX + args.actor,
job["attempt"], "status=%s" % job["status"])
print("refused: job is %s (attachment %s); replay applies only to failed jobs" % (job["status"], job["a_status"]))
return 3
if job["staging_deleted_at"] is not None and job["a_status"] != "ready":
print("refused: staging object already removed by retention; upload the file again", file=sys.stderr)
return 4
await conn.execute(
"""
UPDATE jobs SET status = 'queued', max_attempts = attempt + $2, replays = replays + 1,
error_code = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
WHERE job_id = $1
""", job["job_id"], config.JOB_MAX_ATTEMPTS)
await conn.execute(
"""
UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND status = 'failed'
""", job["attachment_id"])
await jobs.enqueue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1)
await jobs.event(conn, job["job_id"], job["attachment_id"], "replay", ACTOR_PREFIX + args.actor, job["attempt"],
"previous error %s; %d more attempts" % (job["error_code"], config.JOB_MAX_ATTEMPTS))
print("replayed job %s (attempt %d next)" % (job["job_id"], job["attempt"] + 1))
return 0
finally:
await conn.close()
async def cmd_reprocess(args) -> int:
conn = await connect()
try:
async with conn.transaction():
att = await conn.fetchrow("SELECT status, pipeline_version FROM attachments WHERE id = $1 FOR UPDATE",
uuid.UUID(args.attachment_id))
if att is None:
print("unknown attachment", file=sys.stderr)
return 1
if att["pipeline_version"] is not None and att["pipeline_version"] >= args.pipeline_version:
print("refused: v%d already published" % att["pipeline_version"])
return 3
try:
async with conn.transaction():
job_id = await jobs.create_job(conn, uuid.UUID(args.attachment_id), args.pipeline_version,
ACTOR_PREFIX + args.actor)
except asyncpg.UniqueViolationError:
print("refused: a job for this (attachment, pipeline_version) already exists")
return 3
print("created job %s for v%d" % (job_id, args.pipeline_version))
return 0
finally:
await conn.close()
async def inspect(conn, attachment_id: uuid.UUID) -> dict:
att = await conn.fetchrow("SELECT * FROM attachments WHERE id = $1", attachment_id)
if att is None:
return {"attachment": None}
job_rows = await conn.fetch("SELECT * FROM jobs WHERE attachment_id = $1 ORDER BY pipeline_version", attachment_id)
variants = await conn.fetch(
"SELECT pipeline_version, name, object_key, size_bytes, width, height, sha256, attempt FROM attachment_variants "
"WHERE attachment_id = $1 ORDER BY pipeline_version, name", attachment_id)
events = await conn.fetch(
"SELECT event, attempt, actor, detail, created_at FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id)
outbox = await conn.fetch(
"SELECT o.id, o.payload->>'attempt' AS attempt, o.published_at IS NOT NULL AS published, o.publish_count "
"FROM outbox o JOIN jobs j ON j.job_id = o.job_id WHERE j.attachment_id = $1 ORDER BY o.id", attachment_id)
store = ObjectStore()
keys = await asyncio.to_thread(lambda: sorted(k for k, _ in store.list_prefix(FINAL_PREFIX + attachment_id.hex + "/")))
staging = await asyncio.to_thread(lambda: [k for k, _ in store.list_prefix(att["staging_key"] or "-")])
def clean(row):
return {k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v) for k, v in dict(row).items()}
return {
"attachment": {k: v for k, v in clean(att).items() if k in (
"id", "status", "error_code", "pipeline_version", "sha256", "size_bytes", "staging_deleted_at", "updated_at")},
"jobs": [{k: v for k, v in clean(j).items() if k in (
"job_id", "pipeline_version", "status", "attempt", "max_attempts", "error_code", "last_error", "replays",
"lease_owner")} for j in job_rows],
"variants": [clean(v) for v in variants],
"events": [clean(e) for e in events],
"outbox": [clean(o) for o in outbox],
"s3_final_keys": keys,
"s3_staging_keys": staging,
}
async def cmd_inspect(args) -> int:
conn = await connect()
print(json.dumps(await inspect(conn, uuid.UUID(args.attachment_id)), ensure_ascii=False, indent=1))
await conn.close()
return 0
async def cmd_stats(args) -> int:
conn = await connect()
states = {r["status"]: r["n"] for r in await conn.fetch("SELECT status, count(*) AS n FROM jobs GROUP BY status")}
backlog = await conn.fetchval("SELECT count(*) FROM outbox WHERE published_at IS NULL")
await conn.close()
broker = Broker()
await broker.connect()
q = await broker.channel.declare_queue(config.BROKER_QUEUE, passive=True)
d = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True)
out = {"jobs": states, "outbox_unpublished": backlog,
"queue_ready_messages": q.declaration_result.message_count,
"queue_consumers": q.declaration_result.consumer_count,
"dlq_messages": d.declaration_result.message_count}
await broker.close()
print(json.dumps(out))
return 0
async def cmd_fault(args) -> int:
conn = await connect()
if args.fault_cmd == "clear":
n = await conn.execute("DELETE FROM fault_injections")
print("cleared", n.split()[-1])
elif args.fault_cmd == "arm":
if args.point not in faults.POINTS:
print("unknown point; one of: " + ", ".join(sorted(faults.POINTS)), file=sys.stderr)
return 2
if not (args.action in ("crash", "transient", "permanent") or args.action.startswith("delay:")):
print("action: crash | transient | permanent | delay:<seconds>", file=sys.stderr)
return 2
await conn.execute("INSERT INTO fault_injections (point, action, attachment_id, remaining) VALUES ($1, $2, $3, $4)",
args.point, args.action, uuid.UUID(args.attachment) if args.attachment else None, args.count)
print("armed %s -> %s x%d" % (args.point, args.action, args.count))
await conn.close()
return 0
async def cmd_republish(args) -> int:
conn = await connect()
job = await conn.fetchrow(
"SELECT * FROM jobs WHERE attachment_id = $1 AND ($2::int IS NULL OR pipeline_version = $2) "
"ORDER BY pipeline_version LIMIT 1", uuid.UUID(args.attachment_id), args.pipeline_version)
await conn.close()
if job is None:
print("no job", file=sys.stderr)
return 1
attempt = args.attempt or max(1, job["attempt"])
broker = Broker()
await broker.connect()
await broker.publish(job_payload(job["job_id"], job["attachment_id"], job["pipeline_version"], attempt))
await broker.close()
print("published raw message job=%s v%d attempt=%d" % (job["job_id"], job["pipeline_version"], attempt))
return 0
async def cmd_dlq_peek(args) -> int:
broker = Broker()
await broker.connect()
q = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True)
print(json.dumps({"dlq_messages": q.declaration_result.message_count}))
msg = await q.get(no_ack=False, fail=False)
if msg is not None:
print(json.dumps({"body": msg.body.decode("utf-8", "replace")[:500],
"x-death": str((msg.headers or {}).get("x-death"))[:300]}, ensure_ascii=False))
await msg.nack(requeue=True)
await broker.close()
return 0
def main() -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("failed"); s.add_argument("--limit", type=int, default=20); s.set_defaults(fn=cmd_failed)
s = sub.add_parser("replay"); s.add_argument("attachment_id"); s.add_argument("--actor", required=True); s.set_defaults(fn=cmd_replay)
s = sub.add_parser("reprocess"); s.add_argument("attachment_id"); s.add_argument("--pipeline-version", type=int, required=True)
s.add_argument("--actor", default="cli"); s.set_defaults(fn=cmd_reprocess)
s = sub.add_parser("inspect"); s.add_argument("attachment_id"); s.set_defaults(fn=cmd_inspect)
s = sub.add_parser("stats"); s.set_defaults(fn=cmd_stats)
s = sub.add_parser("fault"); fs = s.add_subparsers(dest="fault_cmd", required=True)
a = fs.add_parser("arm"); a.add_argument("point"); a.add_argument("action")
a.add_argument("--attachment"); a.add_argument("--count", type=int, default=1)
fs.add_parser("clear"); s.set_defaults(fn=cmd_fault)
s = sub.add_parser("republish"); s.add_argument("attachment_id"); s.add_argument("--attempt", type=int)
s.add_argument("--pipeline-version", type=int); s.set_defaults(fn=cmd_republish)
s = sub.add_parser("dlq-peek"); s.set_defaults(fn=cmd_dlq_peek)
args = p.parse_args()
return asyncio.run(args.fn(args))
if __name__ == "__main__":
raise SystemExit(main())