Files
mtusi2026lab4/scripts/reconcile.py
T
2026-10-29 02:56:13 +03:00

122 lines
3.5 KiB
Python

import argparse
import asyncio
import json
import sys
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import text
from src.database import SessionFactory, engine
from src.storage import S3_BUCKET, s3_client
def list_s3_objects() -> dict:
objects = {}
paginator = s3_client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=S3_BUCKET):
for item in page.get("Contents", []):
objects[item["Key"]] = item["LastModified"]
return objects
async def list_db_keys() -> set[str]:
async with SessionFactory() as session:
result = await session.execute(text("SELECT s3_key FROM attachments"))
return set(result.scalars().all())
async def db_key_exists(key: str) -> bool:
async with SessionFactory() as session:
result = await session.execute(
text("SELECT 1 FROM attachments WHERE s3_key = :key"),
{"key": key},
)
return result.first() is not None
def inspect_object(key: str):
return s3_client.head_object(Bucket=S3_BUCKET, Key=key)
def remove_object(key: str) -> None:
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
async def reconcile(repair: bool, grace_hours: int) -> int:
started = time.perf_counter()
try:
db_keys, s3_objects = await asyncio.gather(
list_db_keys(),
asyncio.to_thread(list_s3_objects),
)
s3_keys = set(s3_objects)
orphan_keys = sorted(s3_keys - db_keys)
missing_keys = sorted(db_keys - s3_keys)
cutoff = datetime.now(timezone.utc) - timedelta(hours=grace_hours)
deleted = []
skipped = []
if repair:
for key in orphan_keys:
if s3_objects[key] > cutoff:
skipped.append({"key": key, "reason": "grace_period"})
continue
if await db_key_exists(key):
skipped.append({"key": key, "reason": "database_record_exists"})
continue
metadata = await asyncio.to_thread(inspect_object, key)
if metadata["LastModified"] > cutoff:
skipped.append({"key": key, "reason": "object_too_recent"})
continue
if await db_key_exists(key):
skipped.append({"key": key, "reason": "database_record_exists"})
continue
await asyncio.to_thread(remove_object, key)
deleted.append(key)
report = {
"mode": "repair" if repair else "dry-run",
"grace_hours": grace_hours,
"database_objects": len(db_keys),
"storage_objects": len(s3_keys),
"orphan_objects": orphan_keys,
"missing_objects": missing_keys,
"deleted_objects": deleted,
"skipped_objects": skipped,
"duration_seconds": round(time.perf_counter() - started, 4),
}
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
except Exception as error:
print(f"Reconciliation failed: {error}", file=sys.stderr)
return 1
finally:
await engine.dispose()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--repair", action="store_true")
parser.add_argument("--grace-hours", type=int, default=24)
args = parser.parse_args()
if args.grace_hours < 1:
parser.error("--grace-hours must be at least 1")
sys.exit(asyncio.run(reconcile(args.repair, args.grace_hours)))