Files
mtusi2026lab4/scripts/reconcile.py
T

75 lines
2.4 KiB
Python

#!/usr/bin/env python3
"""Find and clean orphan S3 objects older than GRACE; log metadata without object.
Usage (inside api container):
docker compose exec -T api-1 python /tmp/reconcile.py --dry-run
docker compose exec -T api-1 python /tmp/reconcile.py
"""
import argparse
import asyncio
import os
from datetime import datetime, timedelta, timezone
import asyncpg
from minio import Minio
GRACE_HOURS = 1
BUCKET = os.getenv("S3_BUCKET", "attachments")
DB_URL = os.getenv(
"DATABASE_URL",
"postgresql://messenger:messenger@postgres:5432/messenger",
)
S3 = Minio(
os.getenv("S3_ENDPOINT", "minio:9000"),
access_key=os.getenv("S3_ACCESS_KEY", "api-user"),
secret_key=os.getenv("S3_SECRET_KEY", "api-user-secret-123"),
secure=False,
)
async def main(dry_run: bool) -> int:
pool = await asyncpg.create_pool(DB_URL)
try:
async with pool.acquire() as conn:
rows = await conn.fetch("SELECT object_key FROM attachments")
known = {r["object_key"] for r in rows}
cutoff = datetime.now(timezone.utc) - timedelta(hours=GRACE_HOURS)
removed = 0
skipped_fresh = 0
for obj in S3.list_objects(BUCKET, recursive=True):
if obj.object_name in known:
continue
if obj.last_modified and obj.last_modified > cutoff:
skipped_fresh += 1
continue
print(f"orphan: {obj.object_name}")
if not dry_run:
S3.remove_object(BUCKET, obj.object_name)
removed += 1
missing = 0
async with pool.acquire() as conn:
rows = await conn.fetch("SELECT id, object_key FROM attachments")
for r in rows:
try:
S3.stat_object(BUCKET, r["object_key"])
except Exception:
missing += 1
print(f"missing object for attachment {r['id']}: {r['object_key']}")
mode = "dry-run" if dry_run else "apply"
print(
f"reconcile ({mode}): orphans_found={removed} "
f"fresh_skipped={skipped_fresh} metadata_without_object={missing}"
)
return 0
finally:
await pool.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
raise SystemExit(asyncio.run(main(args.dry_run)))