#!/usr/bin/env python3 import os import argparse import boto3 from botocore.client import Config import psycopg DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger") S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000") S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin") S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin") S3_BUCKET = os.environ.get("S3_BUCKET", "attachments") s3 = boto3.client( "s3", endpoint_url=S3_ENDPOINT, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, config=Config(signature_version="s3v4"), region_name="us-east-1" ) def main(): parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.") parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3") args = parser.parse_args() print("Connecting to S3 and PostgreSQL...") # 1. Получаем все ключи из S3 s3_keys = set() paginator = s3.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=S3_BUCKET): for obj in page.get("Contents", []): s3_keys.add(obj["Key"]) # 2. Получаем все s3_key из БД db_keys = set() with psycopg.connect(DATABASE_URL) as conn: with conn.cursor() as cur: cur.execute("SELECT s3_key FROM attachments") for row in cur.fetchall(): db_keys.add(row[0]) # 3. Сравниваем orphans = s3_keys - db_keys missing = db_keys - s3_keys print(f"\n--- RECONCILIATION REPORT ---") print(f"Total objects in S3: {len(s3_keys)}") print(f"Total records in DB: {len(db_keys)}") print(f"Orphan objects (in S3, not in DB): {len(orphans)}") print(f"Missing objects (in DB, not in S3): {len(missing)}") if orphans: print("\n[ORPHANS] (Safe to delete, no metadata in DB):") for key in orphans: print(f" - {key}") if missing: print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):") for key in missing: print(f" - {key}") if args.fix and orphans: print("\n[FIX] Deleting orphan objects from S3...") for key in orphans: s3.delete_object(Bucket=S3_BUCKET, Key=key) print(f" Deleted {key}") if not args.fix and (orphans or missing): print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).") if __name__ == "__main__": main()