123 lines
4.4 KiB
Python
123 lines
4.4 KiB
Python
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
|
|
from src.object_store import ObjectStore
|
|
from src.pipeline import database
|
|
from src.settings import integer
|
|
|
|
|
|
def reconcile(db, store, *, apply=False, grace=None):
|
|
grace = (
|
|
grace
|
|
if grace is not None
|
|
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
|
|
)
|
|
if grace < 300:
|
|
raise ValueError("Retention must be at least 300 seconds")
|
|
actions = []
|
|
for obj in store.objects():
|
|
# Recheck each candidate in a bounded transaction, including late PUT age.
|
|
with db.connection() as conn, conn.transaction():
|
|
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
|
|
"now"
|
|
]
|
|
if (now - obj.last_modified).total_seconds() < grace:
|
|
continue
|
|
reservation = conn.execute(
|
|
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
|
|
(obj.object_name,),
|
|
).fetchone()
|
|
if reservation and reservation["expires_at"] > now:
|
|
continue
|
|
jobs = conn.execute(
|
|
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
|
|
(obj.object_name, obj.object_name),
|
|
).fetchall()
|
|
if any(j["state"] in {"queued", "processing"} for j in jobs):
|
|
continue
|
|
current = conn.execute(
|
|
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
|
|
(obj.object_name,),
|
|
).fetchone()
|
|
if current:
|
|
continue
|
|
if any(
|
|
j["state"] == "failed"
|
|
and (now - j["updated_at"]).total_seconds() < grace
|
|
for j in jobs
|
|
):
|
|
continue
|
|
head = store.head(key=obj.object_name)
|
|
if (
|
|
head is None
|
|
or (now - head.last_modified).total_seconds() < grace
|
|
):
|
|
continue
|
|
action = {
|
|
"action": "delete_expired_object",
|
|
"key": obj.object_name,
|
|
"applied": apply,
|
|
}
|
|
if apply:
|
|
# Taking the reservation lock fences an expired API before deletion.
|
|
if reservation:
|
|
conn.execute(
|
|
"DELETE FROM upload_reservations WHERE id=%s",
|
|
(reservation["id"],),
|
|
)
|
|
store.delete(key=obj.object_name)
|
|
for job in jobs:
|
|
if (
|
|
job["state"] == "failed"
|
|
and job["source_key"] == obj.object_name
|
|
):
|
|
conn.execute(
|
|
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
|
|
(job["id"],),
|
|
)
|
|
conn.execute(
|
|
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
|
|
(job["attachment_id"], job["pipeline_version"]),
|
|
)
|
|
actions.append(action)
|
|
return {"apply": apply, "retention_seconds": grace, "actions": actions}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--apply", action="store_true")
|
|
p.add_argument("--loop", action="store_true")
|
|
a = p.parse_args()
|
|
db, store = database(), ObjectStore()
|
|
try:
|
|
while True:
|
|
try:
|
|
print(
|
|
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
|
|
)
|
|
except Exception:
|
|
if not a.loop:
|
|
raise
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"event": "cleanup_retry",
|
|
"code": "dependency_unavailable",
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
if not a.loop:
|
|
break
|
|
time.sleep(60)
|
|
finally:
|
|
db.close()
|
|
store.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|