Files
mtusi2026lab4/scripts/reconcile.py

70 lines
5.0 KiB
Python
Executable File

#!/usr/bin/env python3
"""Reconcile the private S3 bucket with attachment metadata.
Dry-run is the default. Apply mode only removes old untracked objects and old
unreferenced attachment records after the grace period. Missing tracked objects
are marked failed, never silently recreated. Referenced attachments are never
removed by this tool.
"""
from __future__ import annotations
import argparse, json, os, sys, time
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
from psycopg.rows import dict_row
from botocore.config import Config
def main() -> int:
p=argparse.ArgumentParser(description=__doc__)
p.add_argument("--apply",action="store_true",help="apply safe repairs; default only reports")
p.add_argument("--grace-seconds",type=int,default=86400)
args=p.parse_args()
if args.grace_seconds<3600:
p.error("--grace-seconds must be at least 3600 (one hour)")
start=time.monotonic(); cutoff=datetime.now(timezone.utc)-timedelta(seconds=args.grace_seconds)
s3=boto3.client("s3",endpoint_url=os.getenv("S3_ENDPOINT","http://localhost:9000"),region_name=os.getenv("S3_REGION","us-east-1"),aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],config=Config(connect_timeout=3,read_timeout=15,retries={"max_attempts":2},s3={"addressing_style":"path"}))
conn=psycopg.connect(os.environ["DATABASE_URL"],row_factory=dict_row)
report={"mode":"apply" if args.apply else "dry-run","grace_seconds":args.grace_seconds,"untracked_objects":[],"missing_objects":[],"stale_unreferenced_metadata":[],"deferred_fresh_metadata":[],"actions":[],"elapsed_seconds":None}
try:
attachments=conn.execute("SELECT a.id,a.object_key,a.created_at,a.status,EXISTS(SELECT 1 FROM message_attachments ma WHERE ma.attachment_id=a.id) AS referenced FROM attachments a").fetchall()
by_key={a["object_key"]:a for a in attachments}; seen={}; token=None
while True:
params={"Bucket":os.getenv("S3_BUCKET","messenger-attachments"),"Prefix":"attachments/"}
if token: params["ContinuationToken"]=token
page=s3.list_objects_v2(**params)
for obj in page.get("Contents",[]):
key=obj["Key"]; seen[key]=obj["LastModified"].astimezone(timezone.utc)
if key not in by_key:
record={"key":key,"last_modified":obj["LastModified"].astimezone(timezone.utc).isoformat(),"size_bytes":obj["Size"]}
report["untracked_objects"].append(record)
if args.apply and obj["LastModified"].astimezone(timezone.utc)<cutoff:
s3.delete_object(Bucket=params["Bucket"],Key=key); report["actions"].append({"action":"delete_untracked_object","key":key})
if not page.get("IsTruncated"): break
token=page["NextContinuationToken"]
for a in attachments:
if a["object_key"] not in seen:
report["missing_objects"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
if args.apply and a["status"]!="failed":
conn.execute("UPDATE attachments SET status='failed',error_code='object_missing',updated_at=now() WHERE id=%s",(a["id"],)); report["actions"].append({"action":"mark_missing_failed","attachment_id":str(a["id"])})
object_is_old=a["object_key"] not in seen or seen[a["object_key"]]<cutoff
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and not object_is_old:
report["deferred_fresh_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"reason":"object_is_within_grace_period"})
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and object_is_old:
report["stale_unreferenced_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
if args.apply:
# Recheck reference and age inside the write transaction to avoid racing a message send.
with conn.transaction():
row=conn.execute("SELECT status,created_at,NOT EXISTS(SELECT 1 FROM message_attachments WHERE attachment_id=%s) AS unused FROM attachments WHERE id=%s FOR UPDATE",(a["id"],a["id"])).fetchone()
if row and row["unused"] and row["created_at"].astimezone(timezone.utc)<cutoff:
if a["object_key"] in seen: s3.delete_object(Bucket=os.getenv("S3_BUCKET","messenger-attachments"),Key=a["object_key"])
conn.execute("DELETE FROM attachments WHERE id=%s",(a["id"],)); report["actions"].append({"action":"delete_stale_unreferenced","attachment_id":str(a["id"])})
if args.apply: conn.commit()
report["elapsed_seconds"]=round(time.monotonic()-start,4)
print(json.dumps(report,ensure_ascii=False,sort_keys=True,indent=2,default=str))
return 0
finally:
conn.close()
if __name__=="__main__": sys.exit(main())