75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Find and clean orphan S3 objects older than GRACE; log metadata without object.
|
|
|
|
Usage (inside api container):
|
|
docker compose exec -T api-1 python /tmp/reconcile.py --dry-run
|
|
docker compose exec -T api-1 python /tmp/reconcile.py
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import asyncpg
|
|
from minio import Minio
|
|
|
|
GRACE_HOURS = 1
|
|
BUCKET = os.getenv("S3_BUCKET", "attachments")
|
|
DB_URL = os.getenv(
|
|
"DATABASE_URL",
|
|
"postgresql://messenger:messenger@postgres:5432/messenger",
|
|
)
|
|
S3 = Minio(
|
|
os.getenv("S3_ENDPOINT", "minio:9000"),
|
|
access_key=os.getenv("S3_ACCESS_KEY", "api-user"),
|
|
secret_key=os.getenv("S3_SECRET_KEY", "api-user-secret-123"),
|
|
secure=False,
|
|
)
|
|
|
|
|
|
async def main(dry_run: bool) -> int:
|
|
pool = await asyncpg.create_pool(DB_URL)
|
|
try:
|
|
async with pool.acquire() as conn:
|
|
rows = await conn.fetch("SELECT object_key FROM attachments")
|
|
known = {r["object_key"] for r in rows}
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=GRACE_HOURS)
|
|
removed = 0
|
|
skipped_fresh = 0
|
|
for obj in S3.list_objects(BUCKET, recursive=True):
|
|
if obj.object_name in known:
|
|
continue
|
|
if obj.last_modified and obj.last_modified > cutoff:
|
|
skipped_fresh += 1
|
|
continue
|
|
print(f"orphan: {obj.object_name}")
|
|
if not dry_run:
|
|
S3.remove_object(BUCKET, obj.object_name)
|
|
removed += 1
|
|
|
|
missing = 0
|
|
async with pool.acquire() as conn:
|
|
rows = await conn.fetch("SELECT id, object_key FROM attachments")
|
|
for r in rows:
|
|
try:
|
|
S3.stat_object(BUCKET, r["object_key"])
|
|
except Exception:
|
|
missing += 1
|
|
print(f"missing object for attachment {r['id']}: {r['object_key']}")
|
|
|
|
mode = "dry-run" if dry_run else "apply"
|
|
print(
|
|
f"reconcile ({mode}): orphans_found={removed} "
|
|
f"fresh_skipped={skipped_fresh} metadata_without_object={missing}"
|
|
)
|
|
return 0
|
|
finally:
|
|
await pool.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args()
|
|
raise SystemExit(asyncio.run(main(args.dry_run))) |