Files
mtusi2026lab5/src/messenger/cleanup.py
T

123 lines
5.5 KiB
Python

"""Dry-run by default. Retention GC protects all active jobs and every published object."""
import argparse
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.pipeline import event
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def collect(db, store, retention, apply=False):
if retention < 300:
raise ValueError("Retention must be at least 300 seconds")
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
with db.connection() as lock:
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
return {**counts, "busy": True}
try:
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
for obj in page.get("Contents", []):
counts["scanned"] += 1
key = obj["Key"]
parts = key.split("/")
try:
if (
parts[0] not in ("staging", "objects")
or str(UUID(parts[1])) != parts[1]
):
continue
except ValueError, IndexError:
continue
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
continue
with db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
if (
reservation
and reservation["state"] == "uploading"
and reservation["expires_at"] > datetime.now(UTC)
):
counts["protected"] += 1
continue
# This row lock serializes replay/new-version with deletion eligibility.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
active = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state IN ('queued','processing') LIMIT 1",
(parts[1],),
).fetchone()
published = (
a
and a["storage_state"] == "ready"
and (
key == a["object_key"]
or any(v["object_key"] == key for v in a["variants"])
)
)
recent_failed = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state='failed' "
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
(parts[1], retention),
).fetchone()
if active or published or recent_failed:
counts["protected"] += 1
continue
head = store.head(key)
if (
not head
or head["ETag"] != obj["ETag"]
or head["ContentLength"] != obj["Size"]
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
< retention
):
continue
counts["eligible"] += 1
if apply:
if reservation and reservation["state"] == "uploading":
conn.execute(
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
(parts[1],),
)
store.client.delete_object(Bucket=store.bucket, Key=key)
counts["deleted"] += 1
rows = conn.execute(
"UPDATE attachment_jobs SET source_expired=true WHERE "
"source_key=%s "
"AND state='failed' RETURNING job_id",
(key,),
).fetchall()
for row in rows:
event(conn, row["job_id"], "source_expired")
finally:
lock.execute("SELECT pg_advisory_unlock(2301005)")
return counts
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
store = ObjectStore(settings)
try:
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
finally:
store.close()
db.close()
if __name__ == "__main__":
main()