forked from mtusi2026/mtusi2026lab6
lab 6 observability
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator tool: failed-job registry and SAFE explicit replay (lab5, level 4).
|
||||
|
||||
docker compose exec worker python scripts/replay_failed.py --list
|
||||
docker compose exec worker python scripts/replay_failed.py --job-id <uuid> --reason "ticket-123"
|
||||
|
||||
Replay never executes a duplicate of a successful job: it requires status=failed, that the staging source still exists,
|
||||
and that the effect (all variants of the pipeline version) is NOT already present. Everything happens in ONE transaction
|
||||
(state change + audit event + outbox row), so the new attempt is delivered by the normal dispatcher path.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from src import jobs # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--list", action="store_true")
|
||||
p.add_argument("--job-id")
|
||||
p.add_argument("--reason", default="operator replay")
|
||||
a = p.parse_args()
|
||||
with psycopg.connect(connect_timeout=5) as conn:
|
||||
if a.list or not a.job_id:
|
||||
for r in conn.execute("SELECT job_id, attachment_id, error_code, attempt, updated_at FROM attachment_jobs WHERE status='failed' ORDER BY updated_at"):
|
||||
print(*r, sep="\t")
|
||||
return 0
|
||||
row = conn.execute("SELECT j.status, j.attachment_id, j.pipeline_version, a.staging_key FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j", (a.job_id,)).fetchone()
|
||||
if row is None:
|
||||
print("REFUSED: unknown job"); return 1
|
||||
status, att_id, pv, staging_key = row
|
||||
if status != "failed":
|
||||
print(f"REFUSED: job is {status}, only failed jobs can be replayed"); return 1
|
||||
done = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", (att_id, pv)).fetchone()[0]
|
||||
if done >= len(jobs.VARIANT_ORDER):
|
||||
print("REFUSED: the effect already exists (all variants present)"); return 1
|
||||
if not staging_key:
|
||||
print("REFUSED: staging source expired (retention), upload the file again"); return 1
|
||||
conn.execute("UPDATE attachment_jobs SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (a.job_id,))
|
||||
conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,))
|
||||
jobs.add_event(conn, a.job_id, "replay", a.reason[:200])
|
||||
jobs.enqueue(conn, a.job_id, att_id, pv, 1)
|
||||
print(f"REPLAYED job={a.job_id} attachment={att_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user