Files
mtusi2026lab5/src/reconcile.py
T

83 lines
3.0 KiB
Python

"""Reconcile S3 objects and attachment metadata (lab4 grade-4 safety net).
The synchronous upload spends a short window between the S3 PUT and the
PostgreSQL COMMIT. A crash there can leave an *orphan* object (in S3, no row);
the inline compensation in the API covers the ordinary failure path, and this
tool covers crash windows and any object created but never referenced.
docker compose exec -T api python -m src.reconcile # dry-run
docker compose exec -T api python -m src.reconcile --apply # delete orphans
Safety:
* dry-run by default; ``--apply`` is required to delete anything;
* only objects with no metadata row are candidates (referenced objects are
never touched);
* only objects older than ``--grace-seconds`` (default 3600) are deleted, so a
just-started upload is never mistaken for an orphan;
* objects present in metadata but missing in S3 are reported, never guessed.
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
from . import repositories as repo
from . import storage
from .db import pool
def main() -> None:
parser = argparse.ArgumentParser(description="Reconcile S3 objects and attachment metadata")
parser.add_argument("--apply", action="store_true", help="actually delete orphan objects")
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--limit", type=int, default=50, help="how many orphans to print")
args = parser.parse_args()
with pool().connection() as conn:
rows = repo.list_all_attachments(conn)
variants = conn.execute("select s3_key from attachment_variants").fetchall()
known = {}
for row in rows:
for key in (row["staging_key"], row["s3_key"]):
if key:
known[str(key)] = str(row["id"])
for row in variants:
known[str(row["s3_key"])] = "variant"
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds)
objects = list(storage.list_keys(""))
s3_keys = {item["Key"]: item.get("LastModified") for item in objects}
missing = sorted(key for key in known if key not in s3_keys)
orphans = []
for key, modified in s3_keys.items():
if key in known:
continue
if modified is not None and modified > cutoff:
continue # too fresh: may be an in-flight upload
orphans.append(key)
report = {
"db_attachments": len(known),
"s3_objects": len(s3_keys),
"orphans": len(orphans),
"missing": len(missing),
"apply": bool(args.apply),
"grace_seconds": args.grace_seconds,
}
print(json.dumps(report, ensure_ascii=False))
for key in sorted(orphans)[: args.limit]:
print("orphan:", key)
for key in missing[: args.limit]:
print("missing:", key, "attachment_id=" + known[key])
if args.apply and orphans:
deleted = sum(1 for key in orphans if storage.delete_object(key))
print(json.dumps({"deleted_orphans": deleted}))
if __name__ == "__main__":
main()