124 lines
5.3 KiB
Python
124 lines
5.3 KiB
Python
"""Local operator commands; no public HTTP replay or arbitrary object paths."""
|
|
|
|
import argparse
|
|
import json
|
|
|
|
from .domain import Failure
|
|
from .jobs import Jobs, fault
|
|
from .postgres import PostgresStore
|
|
from .storage import ObjectStorage
|
|
from .worker import cleanup_ready
|
|
|
|
|
|
def cleanup(store, storage, failed_hours=168, intent_seconds=3600):
|
|
if failed_hours < 0 or intent_seconds < 0:
|
|
raise ValueError("Retention must be nonnegative")
|
|
counts = {"ready_staging": 0, "failed_sources": 0, "orphan_intents": 0}
|
|
with store.connection() as conn:
|
|
ready = conn.execute(
|
|
"SELECT id FROM attachments WHERE status='ready' AND stage_key IS NOT NULL"
|
|
).fetchall()
|
|
for row in ready:
|
|
cleanup_ready(store, storage, row["id"])
|
|
counts["ready_staging"] += 1
|
|
with store.connection() as conn:
|
|
candidates = conn.execute(
|
|
"SELECT id FROM attachments WHERE status='failed' AND updated_at<now()-(%s * interval '1 hour')",
|
|
(failed_hours,),
|
|
).fetchall()
|
|
for row in candidates:
|
|
with store.connection() as conn, conn.transaction():
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
|
).fetchone()
|
|
# Recheck under the same row lock used by replay and publication.
|
|
if not a or a["status"] != "failed":
|
|
continue
|
|
current = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
|
|
(a["id"], a["pipeline_version"]),
|
|
).fetchone()
|
|
if not current or current["state"] != "failed" or current["source_key"] is None:
|
|
continue
|
|
old = conn.execute(
|
|
"SELECT updated_at<now()-(%s * interval '1 hour') AS old FROM attachments WHERE id=%s",
|
|
(failed_hours, a["id"]),
|
|
).fetchone()["old"]
|
|
if not old:
|
|
continue
|
|
fault(store, "cleanup_before_delete", a["id"])
|
|
# If cleanup commits first, later replay refuses source_missing. If replay
|
|
# commits first, status is queued and cleanup skips it. DELETE stays locked.
|
|
source = current["source_key"]
|
|
storage.delete(source)
|
|
prefix = f"attachments/{a['id']}/v{a['pipeline_version']}"
|
|
for variant in ("original", "thumbnail", "crop", "optimized"):
|
|
storage.delete(prefix + "/" + variant)
|
|
conn.execute("UPDATE attachment_jobs SET source_key=NULL WHERE id=%s", (current["id"],))
|
|
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (a["id"],))
|
|
Jobs.audit(conn, current["id"], "source_retention_expired", current["attempt"])
|
|
counts["failed_sources"] += 1
|
|
with store.connection() as conn:
|
|
intents = conn.execute(
|
|
"SELECT id FROM upload_intents WHERE created_at<now()-(%s * interval '1 second')",
|
|
(intent_seconds,),
|
|
).fetchall()
|
|
for row in intents:
|
|
with store.connection() as conn, conn.transaction():
|
|
intent = conn.execute(
|
|
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", (row["id"],)
|
|
).fetchone()
|
|
if intent is None:
|
|
continue
|
|
referenced = conn.execute(
|
|
"SELECT 1 FROM attachments WHERE id=%s", (row["id"],)
|
|
).fetchone()
|
|
if referenced is None:
|
|
storage.delete(intent["object_key"])
|
|
counts["orphan_intents"] += 1
|
|
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
|
|
return counts
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
for name in ("replay", "reprocess"):
|
|
command = sub.add_parser(name)
|
|
command.add_argument("attachment_id")
|
|
if name == "reprocess":
|
|
command.add_argument("--version", type=int, required=True)
|
|
sub.add_parser("failed")
|
|
command = sub.add_parser("cleanup")
|
|
command.add_argument("--failed-hours", type=float, default=168)
|
|
command.add_argument("--intent-seconds", type=float, default=3600)
|
|
args = parser.parse_args()
|
|
store, storage = PostgresStore(), ObjectStorage()
|
|
try:
|
|
jobs = Jobs(store)
|
|
if args.command == "replay":
|
|
result = {"job_id": jobs.replay(args.attachment_id, storage)}
|
|
elif args.command == "reprocess":
|
|
result = {"job_id": jobs.reprocess(args.attachment_id, args.version, storage)}
|
|
elif args.command == "cleanup":
|
|
result = cleanup(store, storage, args.failed_hours, args.intent_seconds)
|
|
else:
|
|
with store.connection() as conn:
|
|
result = [
|
|
dict(row)
|
|
for row in conn.execute(
|
|
"SELECT j.id,j.attachment_id,j.attempt,f.error_code,f.replay_count FROM failed_jobs f JOIN attachment_jobs j ON j.id=f.job_id WHERE j.state='failed'"
|
|
).fetchall()
|
|
]
|
|
print(json.dumps(result, default=str))
|
|
except Failure as exc:
|
|
print(json.dumps({"status": exc.status, "error": exc.message}))
|
|
raise SystemExit(1) from None
|
|
finally:
|
|
storage.close()
|
|
store.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|