Files
mtusi2026lab6/src/messenger/jobctl.py
T

124 lines
5.3 KiB
Python

"""Local operator commands; no public HTTP replay or arbitrary object paths."""
import argparse
import json
from .domain import Failure
from .jobs import Jobs, fault
from .postgres import PostgresStore
from .storage import ObjectStorage
from .worker import cleanup_ready
def cleanup(store, storage, failed_hours=168, intent_seconds=3600):
if failed_hours < 0 or intent_seconds < 0:
raise ValueError("Retention must be nonnegative")
counts = {"ready_staging": 0, "failed_sources": 0, "orphan_intents": 0}
with store.connection() as conn:
ready = conn.execute(
"SELECT id FROM attachments WHERE status='ready' AND stage_key IS NOT NULL"
).fetchall()
for row in ready:
cleanup_ready(store, storage, row["id"])
counts["ready_staging"] += 1
with store.connection() as conn:
candidates = conn.execute(
"SELECT id FROM attachments WHERE status='failed' AND updated_at<now()-(%s * interval '1 hour')",
(failed_hours,),
).fetchall()
for row in candidates:
with store.connection() as conn, conn.transaction():
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
# Recheck under the same row lock used by replay and publication.
if not a or a["status"] != "failed":
continue
current = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
(a["id"], a["pipeline_version"]),
).fetchone()
if not current or current["state"] != "failed" or current["source_key"] is None:
continue
old = conn.execute(
"SELECT updated_at<now()-(%s * interval '1 hour') AS old FROM attachments WHERE id=%s",
(failed_hours, a["id"]),
).fetchone()["old"]
if not old:
continue
fault(store, "cleanup_before_delete", a["id"])
# If cleanup commits first, later replay refuses source_missing. If replay
# commits first, status is queued and cleanup skips it. DELETE stays locked.
source = current["source_key"]
storage.delete(source)
prefix = f"attachments/{a['id']}/v{a['pipeline_version']}"
for variant in ("original", "thumbnail", "crop", "optimized"):
storage.delete(prefix + "/" + variant)
conn.execute("UPDATE attachment_jobs SET source_key=NULL WHERE id=%s", (current["id"],))
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (a["id"],))
Jobs.audit(conn, current["id"], "source_retention_expired", current["attempt"])
counts["failed_sources"] += 1
with store.connection() as conn:
intents = conn.execute(
"SELECT id FROM upload_intents WHERE created_at<now()-(%s * interval '1 second')",
(intent_seconds,),
).fetchall()
for row in intents:
with store.connection() as conn, conn.transaction():
intent = conn.execute(
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", (row["id"],)
).fetchone()
if intent is None:
continue
referenced = conn.execute(
"SELECT 1 FROM attachments WHERE id=%s", (row["id"],)
).fetchone()
if referenced is None:
storage.delete(intent["object_key"])
counts["orphan_intents"] += 1
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
return counts
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
for name in ("replay", "reprocess"):
command = sub.add_parser(name)
command.add_argument("attachment_id")
if name == "reprocess":
command.add_argument("--version", type=int, required=True)
sub.add_parser("failed")
command = sub.add_parser("cleanup")
command.add_argument("--failed-hours", type=float, default=168)
command.add_argument("--intent-seconds", type=float, default=3600)
args = parser.parse_args()
store, storage = PostgresStore(), ObjectStorage()
try:
jobs = Jobs(store)
if args.command == "replay":
result = {"job_id": jobs.replay(args.attachment_id, storage)}
elif args.command == "reprocess":
result = {"job_id": jobs.reprocess(args.attachment_id, args.version, storage)}
elif args.command == "cleanup":
result = cleanup(store, storage, args.failed_hours, args.intent_seconds)
else:
with store.connection() as conn:
result = [
dict(row)
for row in conn.execute(
"SELECT j.id,j.attachment_id,j.attempt,f.error_code,f.replay_count FROM failed_jobs f JOIN attachment_jobs j ON j.id=f.job_id WHERE j.state='failed'"
).fetchall()
]
print(json.dumps(result, default=str))
except Failure as exc:
print(json.dumps({"status": exc.status, "error": exc.message}))
raise SystemExit(1) from None
finally:
storage.close()
store.close()
if __name__ == "__main__":
main()