forked from mtusi2026/mtusi2026lab4
70 lines
5.0 KiB
Python
Executable File
70 lines
5.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reconcile the private S3 bucket with attachment metadata.
|
|
|
|
Dry-run is the default. Apply mode only removes old untracked objects and old
|
|
unreferenced attachment records after the grace period. Missing tracked objects
|
|
are marked failed, never silently recreated. Referenced attachments are never
|
|
removed by this tool.
|
|
"""
|
|
from __future__ import annotations
|
|
import argparse, json, os, sys, time
|
|
from datetime import datetime, timedelta, timezone
|
|
import boto3
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
from botocore.config import Config
|
|
|
|
|
|
def main() -> int:
|
|
p=argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--apply",action="store_true",help="apply safe repairs; default only reports")
|
|
p.add_argument("--grace-seconds",type=int,default=86400)
|
|
args=p.parse_args()
|
|
if args.grace_seconds<3600:
|
|
p.error("--grace-seconds must be at least 3600 (one hour)")
|
|
start=time.monotonic(); cutoff=datetime.now(timezone.utc)-timedelta(seconds=args.grace_seconds)
|
|
s3=boto3.client("s3",endpoint_url=os.getenv("S3_ENDPOINT","http://localhost:9000"),region_name=os.getenv("S3_REGION","us-east-1"),aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],config=Config(connect_timeout=3,read_timeout=15,retries={"max_attempts":2},s3={"addressing_style":"path"}))
|
|
conn=psycopg.connect(os.environ["DATABASE_URL"],row_factory=dict_row)
|
|
report={"mode":"apply" if args.apply else "dry-run","grace_seconds":args.grace_seconds,"untracked_objects":[],"missing_objects":[],"stale_unreferenced_metadata":[],"deferred_fresh_metadata":[],"actions":[],"elapsed_seconds":None}
|
|
try:
|
|
attachments=conn.execute("SELECT a.id,a.object_key,a.created_at,a.status,EXISTS(SELECT 1 FROM message_attachments ma WHERE ma.attachment_id=a.id) AS referenced FROM attachments a").fetchall()
|
|
by_key={a["object_key"]:a for a in attachments}; seen={}; token=None
|
|
while True:
|
|
params={"Bucket":os.getenv("S3_BUCKET","messenger-attachments"),"Prefix":"attachments/"}
|
|
if token: params["ContinuationToken"]=token
|
|
page=s3.list_objects_v2(**params)
|
|
for obj in page.get("Contents",[]):
|
|
key=obj["Key"]; seen[key]=obj["LastModified"].astimezone(timezone.utc)
|
|
if key not in by_key:
|
|
record={"key":key,"last_modified":obj["LastModified"].astimezone(timezone.utc).isoformat(),"size_bytes":obj["Size"]}
|
|
report["untracked_objects"].append(record)
|
|
if args.apply and obj["LastModified"].astimezone(timezone.utc)<cutoff:
|
|
s3.delete_object(Bucket=params["Bucket"],Key=key); report["actions"].append({"action":"delete_untracked_object","key":key})
|
|
if not page.get("IsTruncated"): break
|
|
token=page["NextContinuationToken"]
|
|
for a in attachments:
|
|
if a["object_key"] not in seen:
|
|
report["missing_objects"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
|
|
if args.apply and a["status"]!="failed":
|
|
conn.execute("UPDATE attachments SET status='failed',error_code='object_missing',updated_at=now() WHERE id=%s",(a["id"],)); report["actions"].append({"action":"mark_missing_failed","attachment_id":str(a["id"])})
|
|
object_is_old=a["object_key"] not in seen or seen[a["object_key"]]<cutoff
|
|
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and not object_is_old:
|
|
report["deferred_fresh_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"reason":"object_is_within_grace_period"})
|
|
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and object_is_old:
|
|
report["stale_unreferenced_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
|
|
if args.apply:
|
|
# Recheck reference and age inside the write transaction to avoid racing a message send.
|
|
with conn.transaction():
|
|
row=conn.execute("SELECT status,created_at,NOT EXISTS(SELECT 1 FROM message_attachments WHERE attachment_id=%s) AS unused FROM attachments WHERE id=%s FOR UPDATE",(a["id"],a["id"])).fetchone()
|
|
if row and row["unused"] and row["created_at"].astimezone(timezone.utc)<cutoff:
|
|
if a["object_key"] in seen: s3.delete_object(Bucket=os.getenv("S3_BUCKET","messenger-attachments"),Key=a["object_key"])
|
|
conn.execute("DELETE FROM attachments WHERE id=%s",(a["id"],)); report["actions"].append({"action":"delete_stale_unreferenced","attachment_id":str(a["id"])})
|
|
if args.apply: conn.commit()
|
|
report["elapsed_seconds"]=round(time.monotonic()-start,4)
|
|
print(json.dumps(report,ensure_ascii=False,sort_keys=True,indent=2,default=str))
|
|
return 0
|
|
finally:
|
|
conn.close()
|
|
|
|
if __name__=="__main__": sys.exit(main())
|