forked from mtusi2026/mtusi2026lab5
123 lines
5.5 KiB
Python
123 lines
5.5 KiB
Python
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from uuid import UUID
|
|
|
|
from messenger.pipeline import event
|
|
from messenger.repository import PostgresRepository
|
|
from messenger.settings import Settings
|
|
from messenger.storage import ObjectStore
|
|
|
|
|
|
def collect(db, store, retention, apply=False):
|
|
if retention < 300:
|
|
raise ValueError("Retention must be at least 300 seconds")
|
|
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
|
|
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
|
|
with db.connection() as lock:
|
|
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
|
|
return {**counts, "busy": True}
|
|
try:
|
|
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
|
|
for obj in page.get("Contents", []):
|
|
counts["scanned"] += 1
|
|
key = obj["Key"]
|
|
parts = key.split("/")
|
|
try:
|
|
if (
|
|
parts[0] not in ("staging", "objects")
|
|
or str(UUID(parts[1])) != parts[1]
|
|
):
|
|
continue
|
|
except ValueError, IndexError:
|
|
continue
|
|
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
|
|
continue
|
|
with db.connection(write=True) as conn:
|
|
reservation = conn.execute(
|
|
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
|
|
).fetchone()
|
|
if (
|
|
reservation
|
|
and reservation["state"] == "uploading"
|
|
and reservation["expires_at"] > datetime.now(UTC)
|
|
):
|
|
counts["protected"] += 1
|
|
continue
|
|
# This row lock serializes replay/new-version with deletion eligibility.
|
|
a = conn.execute(
|
|
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
|
|
).fetchone()
|
|
active = conn.execute(
|
|
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
|
"state IN ('queued','processing') LIMIT 1",
|
|
(parts[1],),
|
|
).fetchone()
|
|
published = (
|
|
a
|
|
and a["storage_state"] == "ready"
|
|
and (
|
|
key == a["object_key"]
|
|
or any(v["object_key"] == key for v in a["variants"])
|
|
)
|
|
)
|
|
recent_failed = conn.execute(
|
|
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
|
"state='failed' "
|
|
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
|
|
(parts[1], retention),
|
|
).fetchone()
|
|
if active or published or recent_failed:
|
|
counts["protected"] += 1
|
|
continue
|
|
head = store.head(key)
|
|
if (
|
|
not head
|
|
or head["ETag"] != obj["ETag"]
|
|
or head["ContentLength"] != obj["Size"]
|
|
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
|
|
< retention
|
|
):
|
|
continue
|
|
counts["eligible"] += 1
|
|
if apply:
|
|
if reservation and reservation["state"] == "uploading":
|
|
conn.execute(
|
|
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
|
|
(parts[1],),
|
|
)
|
|
store.client.delete_object(Bucket=store.bucket, Key=key)
|
|
counts["deleted"] += 1
|
|
rows = conn.execute(
|
|
"UPDATE attachment_jobs SET source_expired=true WHERE "
|
|
"source_key=%s "
|
|
"AND state='failed' RETURNING job_id",
|
|
(key,),
|
|
).fetchall()
|
|
for row in rows:
|
|
event(conn, row["job_id"], "source_expired")
|
|
finally:
|
|
lock.execute("SELECT pg_advisory_unlock(2301005)")
|
|
return counts
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--apply", action="store_true")
|
|
args = parser.parse_args()
|
|
settings = Settings.from_env()
|
|
db = PostgresRepository(settings.database_url, pool_size=2)
|
|
db.open()
|
|
store = ObjectStore(settings)
|
|
try:
|
|
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
|
|
finally:
|
|
store.close()
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|