122 lines
3.5 KiB
Python
122 lines
3.5 KiB
Python
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import text
|
|
|
|
from src.database import SessionFactory, engine
|
|
from src.storage import S3_BUCKET, s3_client
|
|
|
|
|
|
def list_s3_objects() -> dict:
|
|
objects = {}
|
|
paginator = s3_client.get_paginator("list_objects_v2")
|
|
|
|
for page in paginator.paginate(Bucket=S3_BUCKET):
|
|
for item in page.get("Contents", []):
|
|
objects[item["Key"]] = item["LastModified"]
|
|
|
|
return objects
|
|
|
|
|
|
async def list_db_keys() -> set[str]:
|
|
async with SessionFactory() as session:
|
|
result = await session.execute(text("SELECT s3_key FROM attachments"))
|
|
return set(result.scalars().all())
|
|
|
|
|
|
async def db_key_exists(key: str) -> bool:
|
|
async with SessionFactory() as session:
|
|
result = await session.execute(
|
|
text("SELECT 1 FROM attachments WHERE s3_key = :key"),
|
|
{"key": key},
|
|
)
|
|
return result.first() is not None
|
|
|
|
|
|
def inspect_object(key: str):
|
|
return s3_client.head_object(Bucket=S3_BUCKET, Key=key)
|
|
|
|
|
|
def remove_object(key: str) -> None:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
|
|
|
|
|
|
async def reconcile(repair: bool, grace_hours: int) -> int:
|
|
started = time.perf_counter()
|
|
|
|
try:
|
|
db_keys, s3_objects = await asyncio.gather(
|
|
list_db_keys(),
|
|
asyncio.to_thread(list_s3_objects),
|
|
)
|
|
|
|
s3_keys = set(s3_objects)
|
|
orphan_keys = sorted(s3_keys - db_keys)
|
|
missing_keys = sorted(db_keys - s3_keys)
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=grace_hours)
|
|
deleted = []
|
|
skipped = []
|
|
|
|
if repair:
|
|
for key in orphan_keys:
|
|
if s3_objects[key] > cutoff:
|
|
skipped.append({"key": key, "reason": "grace_period"})
|
|
continue
|
|
|
|
if await db_key_exists(key):
|
|
skipped.append({"key": key, "reason": "database_record_exists"})
|
|
continue
|
|
|
|
metadata = await asyncio.to_thread(inspect_object, key)
|
|
|
|
if metadata["LastModified"] > cutoff:
|
|
skipped.append({"key": key, "reason": "object_too_recent"})
|
|
continue
|
|
|
|
if await db_key_exists(key):
|
|
skipped.append({"key": key, "reason": "database_record_exists"})
|
|
continue
|
|
|
|
await asyncio.to_thread(remove_object, key)
|
|
deleted.append(key)
|
|
|
|
report = {
|
|
"mode": "repair" if repair else "dry-run",
|
|
"grace_hours": grace_hours,
|
|
"database_objects": len(db_keys),
|
|
"storage_objects": len(s3_keys),
|
|
"orphan_objects": orphan_keys,
|
|
"missing_objects": missing_keys,
|
|
"deleted_objects": deleted,
|
|
"skipped_objects": skipped,
|
|
"duration_seconds": round(time.perf_counter() - started, 4),
|
|
}
|
|
|
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
except Exception as error:
|
|
print(f"Reconciliation failed: {error}", file=sys.stderr)
|
|
return 1
|
|
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--repair", action="store_true")
|
|
parser.add_argument("--grace-hours", type=int, default=24)
|
|
args = parser.parse_args()
|
|
|
|
if args.grace_hours < 1:
|
|
parser.error("--grace-hours must be at least 1")
|
|
|
|
sys.exit(asyncio.run(reconcile(args.repair, args.grace_hours)))
|