forked from mtusi2026/mtusi2026lab4
75 lines
2.5 KiB
Python
75 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import argparse
|
|
import boto3
|
|
from botocore.client import Config
|
|
import psycopg
|
|
|
|
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger")
|
|
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000")
|
|
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin")
|
|
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin")
|
|
S3_BUCKET = os.environ.get("S3_BUCKET", "attachments")
|
|
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=S3_ACCESS_KEY,
|
|
aws_secret_access_key=S3_SECRET_KEY,
|
|
config=Config(signature_version="s3v4"),
|
|
region_name="us-east-1"
|
|
)
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.")
|
|
parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3")
|
|
args = parser.parse_args()
|
|
|
|
print("Connecting to S3 and PostgreSQL...")
|
|
|
|
# 1. Получаем все ключи из S3
|
|
s3_keys = set()
|
|
paginator = s3.get_paginator("list_objects_v2")
|
|
for page in paginator.paginate(Bucket=S3_BUCKET):
|
|
for obj in page.get("Contents", []):
|
|
s3_keys.add(obj["Key"])
|
|
|
|
# 2. Получаем все s3_key из БД
|
|
db_keys = set()
|
|
with psycopg.connect(DATABASE_URL) as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT s3_key FROM attachments")
|
|
for row in cur.fetchall():
|
|
db_keys.add(row[0])
|
|
|
|
# 3. Сравниваем
|
|
orphans = s3_keys - db_keys
|
|
missing = db_keys - s3_keys
|
|
|
|
print(f"\n--- RECONCILIATION REPORT ---")
|
|
print(f"Total objects in S3: {len(s3_keys)}")
|
|
print(f"Total records in DB: {len(db_keys)}")
|
|
print(f"Orphan objects (in S3, not in DB): {len(orphans)}")
|
|
print(f"Missing objects (in DB, not in S3): {len(missing)}")
|
|
|
|
if orphans:
|
|
print("\n[ORPHANS] (Safe to delete, no metadata in DB):")
|
|
for key in orphans:
|
|
print(f" - {key}")
|
|
|
|
if missing:
|
|
print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):")
|
|
for key in missing:
|
|
print(f" - {key}")
|
|
|
|
if args.fix and orphans:
|
|
print("\n[FIX] Deleting orphan objects from S3...")
|
|
for key in orphans:
|
|
s3.delete_object(Bucket=S3_BUCKET, Key=key)
|
|
print(f" Deleted {key}")
|
|
|
|
if not args.fix and (orphans or missing):
|
|
print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).")
|
|
|
|
if __name__ == "__main__":
|
|
main() |