54 lines
2.7 KiB
Python
54 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Operator tool: failed-job registry and SAFE explicit replay (lab5, level 4).
|
|
|
|
docker compose exec worker python scripts/replay_failed.py --list
|
|
docker compose exec worker python scripts/replay_failed.py --job-id <uuid> --reason "ticket-123"
|
|
|
|
Replay never executes a duplicate of a successful job: it requires status=failed, that the staging source still exists,
|
|
and that the effect (all variants of the pipeline version) is NOT already present. Everything happens in ONE transaction
|
|
(state change + audit event + outbox row), so the new attempt is delivered by the normal dispatcher path.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
|
|
import psycopg
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from src import jobs # noqa: E402
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--list", action="store_true")
|
|
p.add_argument("--job-id")
|
|
p.add_argument("--reason", default="operator replay")
|
|
a = p.parse_args()
|
|
with psycopg.connect(connect_timeout=5) as conn:
|
|
if a.list or not a.job_id:
|
|
for r in conn.execute("SELECT job_id, attachment_id, error_code, attempt, updated_at FROM attachment_jobs WHERE status='failed' ORDER BY updated_at"):
|
|
print(*r, sep="\t")
|
|
return 0
|
|
row = conn.execute("SELECT j.status, j.attachment_id, j.pipeline_version, a.staging_key FROM attachment_jobs j "
|
|
"JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j", (a.job_id,)).fetchone()
|
|
if row is None:
|
|
print("REFUSED: unknown job"); return 1
|
|
status, att_id, pv, staging_key = row
|
|
if status != "failed":
|
|
print(f"REFUSED: job is {status}, only failed jobs can be replayed"); return 1
|
|
done = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", (att_id, pv)).fetchone()[0]
|
|
if done >= len(jobs.VARIANT_ORDER):
|
|
print("REFUSED: the effect already exists (all variants present)"); return 1
|
|
if not staging_key:
|
|
print("REFUSED: staging source expired (retention), upload the file again"); return 1
|
|
conn.execute("UPDATE attachment_jobs SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (a.job_id,))
|
|
conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,))
|
|
jobs.add_event(conn, a.job_id, "replay", a.reason[:200])
|
|
jobs.enqueue(conn, a.job_id, att_id, pv, 1)
|
|
print(f"REPLAYED job={a.job_id} attachment={att_id}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|