forked from mtusi2026/mtusi2026lab5
83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
"""Reconcile S3 objects and attachment metadata (lab4 grade-4 safety net).
|
|
|
|
The synchronous upload spends a short window between the S3 PUT and the
|
|
PostgreSQL COMMIT. A crash there can leave an *orphan* object (in S3, no row);
|
|
the inline compensation in the API covers the ordinary failure path, and this
|
|
tool covers crash windows and any object created but never referenced.
|
|
|
|
docker compose exec -T api python -m src.reconcile # dry-run
|
|
docker compose exec -T api python -m src.reconcile --apply # delete orphans
|
|
|
|
Safety:
|
|
* dry-run by default; ``--apply`` is required to delete anything;
|
|
* only objects with no metadata row are candidates (referenced objects are
|
|
never touched);
|
|
* only objects older than ``--grace-seconds`` (default 3600) are deleted, so a
|
|
just-started upload is never mistaken for an orphan;
|
|
* objects present in metadata but missing in S3 are reported, never guessed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
|
|
from . import repositories as repo
|
|
from . import storage
|
|
from .db import pool
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Reconcile S3 objects and attachment metadata")
|
|
parser.add_argument("--apply", action="store_true", help="actually delete orphan objects")
|
|
parser.add_argument("--grace-seconds", type=int, default=3600)
|
|
parser.add_argument("--limit", type=int, default=50, help="how many orphans to print")
|
|
args = parser.parse_args()
|
|
|
|
with pool().connection() as conn:
|
|
rows = repo.list_all_attachments(conn)
|
|
variants = conn.execute("select s3_key from attachment_variants").fetchall()
|
|
known = {}
|
|
for row in rows:
|
|
for key in (row["staging_key"], row["s3_key"]):
|
|
if key:
|
|
known[str(key)] = str(row["id"])
|
|
for row in variants:
|
|
known[str(row["s3_key"])] = "variant"
|
|
|
|
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds)
|
|
objects = list(storage.list_keys(""))
|
|
s3_keys = {item["Key"]: item.get("LastModified") for item in objects}
|
|
|
|
missing = sorted(key for key in known if key not in s3_keys)
|
|
orphans = []
|
|
for key, modified in s3_keys.items():
|
|
if key in known:
|
|
continue
|
|
if modified is not None and modified > cutoff:
|
|
continue # too fresh: may be an in-flight upload
|
|
orphans.append(key)
|
|
|
|
report = {
|
|
"db_attachments": len(known),
|
|
"s3_objects": len(s3_keys),
|
|
"orphans": len(orphans),
|
|
"missing": len(missing),
|
|
"apply": bool(args.apply),
|
|
"grace_seconds": args.grace_seconds,
|
|
}
|
|
print(json.dumps(report, ensure_ascii=False))
|
|
|
|
for key in sorted(orphans)[: args.limit]:
|
|
print("orphan:", key)
|
|
for key in missing[: args.limit]:
|
|
print("missing:", key, "attachment_id=" + known[key])
|
|
|
|
if args.apply and orphans:
|
|
deleted = sum(1 for key in orphans if storage.delete_object(key))
|
|
print(json.dumps({"deleted_orphans": deleted}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|