forked from mtusi2026/mtusi2026lab5
254 lines
12 KiB
Python
254 lines
12 KiB
Python
"""Operator CLI (runs inside the worker image, never exposed over HTTP).
|
|
|
|
python -m app.ops failed [--limit N] registry of failed jobs (the "DLQ" of the DB)
|
|
python -m app.ops replay <attachment_id> --actor NAME explicit failed -> queued, audited
|
|
python -m app.ops reprocess <attachment_id> --pipeline-version N new job for a newer pipeline version
|
|
python -m app.ops inspect <attachment_id> JSON: attachment, jobs, variants, events, S3 keys
|
|
python -m app.ops stats queue depth, DLQ, outbox backlog, job states
|
|
python -m app.ops fault arm <point> <action> [--attachment ID] [--count N]
|
|
python -m app.ops fault clear
|
|
python -m app.ops republish <attachment_id> [--attempt N] [--pipeline-version N]
|
|
raw publish bypassing the outbox: duplicate / stale / reordered message for fault tests
|
|
python -m app.ops dlq-peek show (and requeue nothing from) the broker DLQ
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
import uuid
|
|
|
|
import asyncpg
|
|
|
|
from app import config, faults, jobs
|
|
from app.broker import Broker, job_payload
|
|
from app.objects import FINAL_PREFIX, ObjectStore
|
|
|
|
ACTOR_PREFIX = "operator:"
|
|
|
|
|
|
def _ts(v):
|
|
return v.isoformat() if v is not None else None
|
|
|
|
|
|
async def connect():
|
|
return await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
|
|
|
|
|
|
async def cmd_failed(args) -> int:
|
|
conn = await connect()
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, j.last_error,
|
|
j.replays, j.updated_at, a.staging_deleted_at IS NULL AS staging_present
|
|
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT $1
|
|
""", args.limit)
|
|
print(json.dumps([{k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v)
|
|
for k, v in dict(r).items()} for r in rows], ensure_ascii=False, indent=1))
|
|
await conn.close()
|
|
return 0
|
|
|
|
|
|
async def cmd_replay(args) -> int:
|
|
"""failed -> queued. Refuses anything that is not failed: a ready job is never executed again."""
|
|
if not args.actor:
|
|
print("--actor is required for the audit log", file=sys.stderr)
|
|
return 2
|
|
conn = await connect()
|
|
try:
|
|
async with conn.transaction():
|
|
job = await conn.fetchrow(
|
|
"""
|
|
SELECT j.*, a.status AS a_status, a.staging_deleted_at FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.attachment_id = $1 ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a
|
|
""", uuid.UUID(args.attachment_id))
|
|
if job is None:
|
|
print("no job for this attachment", file=sys.stderr)
|
|
return 1
|
|
if job["status"] != "failed":
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"], "replay_refused", ACTOR_PREFIX + args.actor,
|
|
job["attempt"], "status=%s" % job["status"])
|
|
print("refused: job is %s (attachment %s); replay applies only to failed jobs" % (job["status"], job["a_status"]))
|
|
return 3
|
|
if job["staging_deleted_at"] is not None and job["a_status"] != "ready":
|
|
print("refused: staging object already removed by retention; upload the file again", file=sys.stderr)
|
|
return 4
|
|
await conn.execute(
|
|
"""
|
|
UPDATE jobs SET status = 'queued', max_attempts = attempt + $2, replays = replays + 1,
|
|
error_code = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
|
|
WHERE job_id = $1
|
|
""", job["job_id"], config.JOB_MAX_ATTEMPTS)
|
|
await conn.execute(
|
|
"""
|
|
UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = $1 AND status = 'failed'
|
|
""", job["attachment_id"])
|
|
await jobs.enqueue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1)
|
|
await jobs.event(conn, job["job_id"], job["attachment_id"], "replay", ACTOR_PREFIX + args.actor, job["attempt"],
|
|
"previous error %s; %d more attempts" % (job["error_code"], config.JOB_MAX_ATTEMPTS))
|
|
print("replayed job %s (attempt %d next)" % (job["job_id"], job["attempt"] + 1))
|
|
return 0
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
async def cmd_reprocess(args) -> int:
|
|
conn = await connect()
|
|
try:
|
|
async with conn.transaction():
|
|
att = await conn.fetchrow("SELECT status, pipeline_version FROM attachments WHERE id = $1 FOR UPDATE",
|
|
uuid.UUID(args.attachment_id))
|
|
if att is None:
|
|
print("unknown attachment", file=sys.stderr)
|
|
return 1
|
|
if att["pipeline_version"] is not None and att["pipeline_version"] >= args.pipeline_version:
|
|
print("refused: v%d already published" % att["pipeline_version"])
|
|
return 3
|
|
try:
|
|
async with conn.transaction():
|
|
job_id = await jobs.create_job(conn, uuid.UUID(args.attachment_id), args.pipeline_version,
|
|
ACTOR_PREFIX + args.actor)
|
|
except asyncpg.UniqueViolationError:
|
|
print("refused: a job for this (attachment, pipeline_version) already exists")
|
|
return 3
|
|
print("created job %s for v%d" % (job_id, args.pipeline_version))
|
|
return 0
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
async def inspect(conn, attachment_id: uuid.UUID) -> dict:
|
|
att = await conn.fetchrow("SELECT * FROM attachments WHERE id = $1", attachment_id)
|
|
if att is None:
|
|
return {"attachment": None}
|
|
job_rows = await conn.fetch("SELECT * FROM jobs WHERE attachment_id = $1 ORDER BY pipeline_version", attachment_id)
|
|
variants = await conn.fetch(
|
|
"SELECT pipeline_version, name, object_key, size_bytes, width, height, sha256, attempt FROM attachment_variants "
|
|
"WHERE attachment_id = $1 ORDER BY pipeline_version, name", attachment_id)
|
|
events = await conn.fetch(
|
|
"SELECT event, attempt, actor, detail, created_at FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id)
|
|
outbox = await conn.fetch(
|
|
"SELECT o.id, o.payload->>'attempt' AS attempt, o.published_at IS NOT NULL AS published, o.publish_count "
|
|
"FROM outbox o JOIN jobs j ON j.job_id = o.job_id WHERE j.attachment_id = $1 ORDER BY o.id", attachment_id)
|
|
store = ObjectStore()
|
|
keys = await asyncio.to_thread(lambda: sorted(k for k, _ in store.list_prefix(FINAL_PREFIX + attachment_id.hex + "/")))
|
|
staging = await asyncio.to_thread(lambda: [k for k, _ in store.list_prefix(att["staging_key"] or "-")])
|
|
|
|
def clean(row):
|
|
return {k: (str(v) if isinstance(v, uuid.UUID) else _ts(v) if hasattr(v, "isoformat") else v) for k, v in dict(row).items()}
|
|
return {
|
|
"attachment": {k: v for k, v in clean(att).items() if k in (
|
|
"id", "status", "error_code", "pipeline_version", "sha256", "size_bytes", "staging_deleted_at", "updated_at")},
|
|
"jobs": [{k: v for k, v in clean(j).items() if k in (
|
|
"job_id", "pipeline_version", "status", "attempt", "max_attempts", "error_code", "last_error", "replays",
|
|
"lease_owner")} for j in job_rows],
|
|
"variants": [clean(v) for v in variants],
|
|
"events": [clean(e) for e in events],
|
|
"outbox": [clean(o) for o in outbox],
|
|
"s3_final_keys": keys,
|
|
"s3_staging_keys": staging,
|
|
}
|
|
|
|
|
|
async def cmd_inspect(args) -> int:
|
|
conn = await connect()
|
|
print(json.dumps(await inspect(conn, uuid.UUID(args.attachment_id)), ensure_ascii=False, indent=1))
|
|
await conn.close()
|
|
return 0
|
|
|
|
|
|
async def cmd_stats(args) -> int:
|
|
conn = await connect()
|
|
states = {r["status"]: r["n"] for r in await conn.fetch("SELECT status, count(*) AS n FROM jobs GROUP BY status")}
|
|
backlog = await conn.fetchval("SELECT count(*) FROM outbox WHERE published_at IS NULL")
|
|
await conn.close()
|
|
broker = Broker()
|
|
await broker.connect()
|
|
q = await broker.channel.declare_queue(config.BROKER_QUEUE, passive=True)
|
|
d = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True)
|
|
out = {"jobs": states, "outbox_unpublished": backlog,
|
|
"queue_ready_messages": q.declaration_result.message_count,
|
|
"queue_consumers": q.declaration_result.consumer_count,
|
|
"dlq_messages": d.declaration_result.message_count}
|
|
await broker.close()
|
|
print(json.dumps(out))
|
|
return 0
|
|
|
|
|
|
async def cmd_fault(args) -> int:
|
|
conn = await connect()
|
|
if args.fault_cmd == "clear":
|
|
n = await conn.execute("DELETE FROM fault_injections")
|
|
print("cleared", n.split()[-1])
|
|
elif args.fault_cmd == "arm":
|
|
if args.point not in faults.POINTS:
|
|
print("unknown point; one of: " + ", ".join(sorted(faults.POINTS)), file=sys.stderr)
|
|
return 2
|
|
if not (args.action in ("crash", "transient", "permanent") or args.action.startswith("delay:")):
|
|
print("action: crash | transient | permanent | delay:<seconds>", file=sys.stderr)
|
|
return 2
|
|
await conn.execute("INSERT INTO fault_injections (point, action, attachment_id, remaining) VALUES ($1, $2, $3, $4)",
|
|
args.point, args.action, uuid.UUID(args.attachment) if args.attachment else None, args.count)
|
|
print("armed %s -> %s x%d" % (args.point, args.action, args.count))
|
|
await conn.close()
|
|
return 0
|
|
|
|
|
|
async def cmd_republish(args) -> int:
|
|
conn = await connect()
|
|
job = await conn.fetchrow(
|
|
"SELECT * FROM jobs WHERE attachment_id = $1 AND ($2::int IS NULL OR pipeline_version = $2) "
|
|
"ORDER BY pipeline_version LIMIT 1", uuid.UUID(args.attachment_id), args.pipeline_version)
|
|
await conn.close()
|
|
if job is None:
|
|
print("no job", file=sys.stderr)
|
|
return 1
|
|
attempt = args.attempt or max(1, job["attempt"])
|
|
broker = Broker()
|
|
await broker.connect()
|
|
await broker.publish(job_payload(job["job_id"], job["attachment_id"], job["pipeline_version"], attempt))
|
|
await broker.close()
|
|
print("published raw message job=%s v%d attempt=%d" % (job["job_id"], job["pipeline_version"], attempt))
|
|
return 0
|
|
|
|
|
|
async def cmd_dlq_peek(args) -> int:
|
|
broker = Broker()
|
|
await broker.connect()
|
|
q = await broker.channel.declare_queue(config.BROKER_DLQ, passive=True)
|
|
print(json.dumps({"dlq_messages": q.declaration_result.message_count}))
|
|
msg = await q.get(no_ack=False, fail=False)
|
|
if msg is not None:
|
|
print(json.dumps({"body": msg.body.decode("utf-8", "replace")[:500],
|
|
"x-death": str((msg.headers or {}).get("x-death"))[:300]}, ensure_ascii=False))
|
|
await msg.nack(requeue=True)
|
|
await broker.close()
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
s = sub.add_parser("failed"); s.add_argument("--limit", type=int, default=20); s.set_defaults(fn=cmd_failed)
|
|
s = sub.add_parser("replay"); s.add_argument("attachment_id"); s.add_argument("--actor", required=True); s.set_defaults(fn=cmd_replay)
|
|
s = sub.add_parser("reprocess"); s.add_argument("attachment_id"); s.add_argument("--pipeline-version", type=int, required=True)
|
|
s.add_argument("--actor", default="cli"); s.set_defaults(fn=cmd_reprocess)
|
|
s = sub.add_parser("inspect"); s.add_argument("attachment_id"); s.set_defaults(fn=cmd_inspect)
|
|
s = sub.add_parser("stats"); s.set_defaults(fn=cmd_stats)
|
|
s = sub.add_parser("fault"); fs = s.add_subparsers(dest="fault_cmd", required=True)
|
|
a = fs.add_parser("arm"); a.add_argument("point"); a.add_argument("action")
|
|
a.add_argument("--attachment"); a.add_argument("--count", type=int, default=1)
|
|
fs.add_parser("clear"); s.set_defaults(fn=cmd_fault)
|
|
s = sub.add_parser("republish"); s.add_argument("attachment_id"); s.add_argument("--attempt", type=int)
|
|
s.add_argument("--pipeline-version", type=int); s.set_defaults(fn=cmd_republish)
|
|
s = sub.add_parser("dlq-peek"); s.set_defaults(fn=cmd_dlq_peek)
|
|
args = p.parse_args()
|
|
return asyncio.run(args.fn(args))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|