Files
mtusi2026lab7/scripts/replay_failed_job.py

159 lines
4.3 KiB
Python

"""Safely replay a failed attachment processing job.
Usage:
python scripts/replay_failed_job.py ATTACHMENT_UUID
"""
import argparse
import logging
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
# Add project root to Python import path.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
from sqlalchemy import select
from app.db.models import Attachment, OutboxEvent, ProcessingJob
from app.db.session import SessionLocal
logger = logging.getLogger("replay_failed_job")
EVENT_TYPE = "attachment.process.v1"
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def replay_failed_job(attachment_id: uuid.UUID) -> None:
db = SessionLocal()
try:
# Lock the attachment first to serialize concurrent replay requests.
attachment = db.scalar(
select(Attachment)
.where(Attachment.id == attachment_id)
.with_for_update()
)
if attachment is None:
raise ValueError(f"Attachment not found: {attachment_id}")
job = db.scalar(
select(ProcessingJob)
.where(
ProcessingJob.attachment_id == attachment_id,
ProcessingJob.pipeline_version == attachment.pipeline_version,
)
.with_for_update()
)
if job is None:
raise ValueError(
f"Processing job not found for attachment: {attachment_id}"
)
# Only terminally failed jobs may be replayed.
if job.status != "failed":
raise ValueError(
f"Job cannot be replayed: current job status is '{job.status}'"
)
if attachment.status != "failed":
raise ValueError(
"Attachment and job states are inconsistent: "
f"attachment status is '{attachment.status}'"
)
# The worker needs the original staged file.
if not attachment.staging_key:
raise ValueError(
"Cannot replay: staging_key is empty. "
"The original file is not available for processing."
)
now = utc_now()
# Reset the existing job. Keep its ID and pipeline version.
job.attempt = 0
job.status = "queued"
job.available_at = now
job.lease_until = None
job.worker_id = None
job.last_error_code = None
# Return attachment metadata to the queued state.
attachment.status = "queued"
attachment.error_code = None
# Create a new durable outbox event in the same transaction.
event = OutboxEvent(
event_type=EVENT_TYPE,
aggregate_id=attachment.id,
payload={
"schema_version": 1,
"job_id": str(job.id),
"type": EVENT_TYPE,
"attachment_id": str(attachment.id),
"pipeline_version": job.pipeline_version,
"attempt": 1,
"created_at": now.isoformat(),
},
status="pending",
)
db.add(event)
db.commit()
print("Replay request successfully committed.")
print(f"Attachment ID: {attachment.id}")
print(f"Job ID: {job.id}")
print(f"Job status: {job.status}")
print(f"Attempt: {job.attempt}")
print(f"Outbox event: {event.id}")
print("Outbox status: pending")
print()
print("The existing outbox-publisher will publish the event.")
except Exception:
db.rollback()
raise
finally:
db.close()
def main() -> int:
parser = argparse.ArgumentParser(
description="Replay a terminally failed attachment processing job."
)
parser.add_argument(
"attachment_id",
help="UUID of the attachment to replay",
)
args = parser.parse_args()
try:
attachment_id = uuid.UUID(args.attachment_id)
replay_failed_job(attachment_id)
return 0
except ValueError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
except Exception:
logger.exception("Could not replay failed processing job")
return 1
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
raise SystemExit(main())