Files
mtusi2026lab4/scripts/reconcile.py
T

75 lines
2.5 KiB
Python

#!/usr/bin/env python3
import os
import argparse
import boto3
from botocore.client import Config
import psycopg
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger")
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000")
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin")
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin")
S3_BUCKET = os.environ.get("S3_BUCKET", "attachments")
s3 = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
def main():
parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.")
parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3")
args = parser.parse_args()
print("Connecting to S3 and PostgreSQL...")
# 1. Получаем все ключи из S3
s3_keys = set()
paginator = s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=S3_BUCKET):
for obj in page.get("Contents", []):
s3_keys.add(obj["Key"])
# 2. Получаем все s3_key из БД
db_keys = set()
with psycopg.connect(DATABASE_URL) as conn:
with conn.cursor() as cur:
cur.execute("SELECT s3_key FROM attachments")
for row in cur.fetchall():
db_keys.add(row[0])
# 3. Сравниваем
orphans = s3_keys - db_keys
missing = db_keys - s3_keys
print(f"\n--- RECONCILIATION REPORT ---")
print(f"Total objects in S3: {len(s3_keys)}")
print(f"Total records in DB: {len(db_keys)}")
print(f"Orphan objects (in S3, not in DB): {len(orphans)}")
print(f"Missing objects (in DB, not in S3): {len(missing)}")
if orphans:
print("\n[ORPHANS] (Safe to delete, no metadata in DB):")
for key in orphans:
print(f" - {key}")
if missing:
print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):")
for key in missing:
print(f" - {key}")
if args.fix and orphans:
print("\n[FIX] Deleting orphan objects from S3...")
for key in orphans:
s3.delete_object(Bucket=S3_BUCKET, Key=key)
print(f" Deleted {key}")
if not args.fix and (orphans or missing):
print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).")
if __name__ == "__main__":
main()