forked from mtusi2026/mtusi2026lab7
159 lines
4.3 KiB
Python
159 lines
4.3 KiB
Python
"""Safely replay a failed attachment processing job.
|
|
|
|
Usage:
|
|
python scripts/replay_failed_job.py ATTACHMENT_UUID
|
|
"""
|
|
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# Add project root to Python import path.
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from sqlalchemy import select
|
|
|
|
from app.db.models import Attachment, OutboxEvent, ProcessingJob
|
|
from app.db.session import SessionLocal
|
|
|
|
|
|
logger = logging.getLogger("replay_failed_job")
|
|
|
|
EVENT_TYPE = "attachment.process.v1"
|
|
|
|
|
|
def utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def replay_failed_job(attachment_id: uuid.UUID) -> None:
|
|
db = SessionLocal()
|
|
|
|
try:
|
|
# Lock the attachment first to serialize concurrent replay requests.
|
|
attachment = db.scalar(
|
|
select(Attachment)
|
|
.where(Attachment.id == attachment_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
if attachment is None:
|
|
raise ValueError(f"Attachment not found: {attachment_id}")
|
|
|
|
job = db.scalar(
|
|
select(ProcessingJob)
|
|
.where(
|
|
ProcessingJob.attachment_id == attachment_id,
|
|
ProcessingJob.pipeline_version == attachment.pipeline_version,
|
|
)
|
|
.with_for_update()
|
|
)
|
|
|
|
if job is None:
|
|
raise ValueError(
|
|
f"Processing job not found for attachment: {attachment_id}"
|
|
)
|
|
|
|
# Only terminally failed jobs may be replayed.
|
|
if job.status != "failed":
|
|
raise ValueError(
|
|
f"Job cannot be replayed: current job status is '{job.status}'"
|
|
)
|
|
|
|
if attachment.status != "failed":
|
|
raise ValueError(
|
|
"Attachment and job states are inconsistent: "
|
|
f"attachment status is '{attachment.status}'"
|
|
)
|
|
|
|
# The worker needs the original staged file.
|
|
if not attachment.staging_key:
|
|
raise ValueError(
|
|
"Cannot replay: staging_key is empty. "
|
|
"The original file is not available for processing."
|
|
)
|
|
|
|
now = utc_now()
|
|
|
|
# Reset the existing job. Keep its ID and pipeline version.
|
|
job.attempt = 0
|
|
job.status = "queued"
|
|
job.available_at = now
|
|
job.lease_until = None
|
|
job.worker_id = None
|
|
job.last_error_code = None
|
|
|
|
# Return attachment metadata to the queued state.
|
|
attachment.status = "queued"
|
|
attachment.error_code = None
|
|
|
|
# Create a new durable outbox event in the same transaction.
|
|
event = OutboxEvent(
|
|
event_type=EVENT_TYPE,
|
|
aggregate_id=attachment.id,
|
|
payload={
|
|
"schema_version": 1,
|
|
"job_id": str(job.id),
|
|
"type": EVENT_TYPE,
|
|
"attachment_id": str(attachment.id),
|
|
"pipeline_version": job.pipeline_version,
|
|
"attempt": 1,
|
|
"created_at": now.isoformat(),
|
|
},
|
|
status="pending",
|
|
)
|
|
|
|
db.add(event)
|
|
db.commit()
|
|
|
|
print("Replay request successfully committed.")
|
|
print(f"Attachment ID: {attachment.id}")
|
|
print(f"Job ID: {job.id}")
|
|
print(f"Job status: {job.status}")
|
|
print(f"Attempt: {job.attempt}")
|
|
print(f"Outbox event: {event.id}")
|
|
print("Outbox status: pending")
|
|
print()
|
|
print("The existing outbox-publisher will publish the event.")
|
|
|
|
except Exception:
|
|
db.rollback()
|
|
raise
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Replay a terminally failed attachment processing job."
|
|
)
|
|
parser.add_argument(
|
|
"attachment_id",
|
|
help="UUID of the attachment to replay",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
attachment_id = uuid.UUID(args.attachment_id)
|
|
replay_failed_job(attachment_id)
|
|
return 0
|
|
|
|
except ValueError as exc:
|
|
print(f"ERROR: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
except Exception:
|
|
logger.exception("Could not replay failed processing job")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO)
|
|
raise SystemExit(main())
|