Files
mtusi2026lab6/src/reconcile.py
T

123 lines
4.4 KiB
Python

"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
import argparse
import json
import time
from src.object_store import ObjectStore
from src.pipeline import database
from src.settings import integer
def reconcile(db, store, *, apply=False, grace=None):
grace = (
grace
if grace is not None
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
)
if grace < 300:
raise ValueError("Retention must be at least 300 seconds")
actions = []
for obj in store.objects():
# Recheck each candidate in a bounded transaction, including late PUT age.
with db.connection() as conn, conn.transaction():
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
"now"
]
if (now - obj.last_modified).total_seconds() < grace:
continue
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
(obj.object_name,),
).fetchone()
if reservation and reservation["expires_at"] > now:
continue
jobs = conn.execute(
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
(obj.object_name, obj.object_name),
).fetchall()
if any(j["state"] in {"queued", "processing"} for j in jobs):
continue
current = conn.execute(
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
(obj.object_name,),
).fetchone()
if current:
continue
if any(
j["state"] == "failed"
and (now - j["updated_at"]).total_seconds() < grace
for j in jobs
):
continue
head = store.head(key=obj.object_name)
if (
head is None
or (now - head.last_modified).total_seconds() < grace
):
continue
action = {
"action": "delete_expired_object",
"key": obj.object_name,
"applied": apply,
}
if apply:
# Taking the reservation lock fences an expired API before deletion.
if reservation:
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s",
(reservation["id"],),
)
store.delete(key=obj.object_name)
for job in jobs:
if (
job["state"] == "failed"
and job["source_key"] == obj.object_name
):
conn.execute(
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
(job["id"],),
)
conn.execute(
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
(job["attachment_id"], job["pipeline_version"]),
)
actions.append(action)
return {"apply": apply, "retention_seconds": grace, "actions": actions}
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--loop", action="store_true")
a = p.parse_args()
db, store = database(), ObjectStore()
try:
while True:
try:
print(
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
)
except Exception:
if not a.loop:
raise
print(
json.dumps(
{
"event": "cleanup_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
if not a.loop:
break
time.sleep(60)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()