ЛР4 - БВТ2302 - Никитина - на 5
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import argparse
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
import psycopg
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger")
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000")
|
||||
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin")
|
||||
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "attachments")
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.")
|
||||
parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Connecting to S3 and PostgreSQL...")
|
||||
|
||||
# 1. Получаем все ключи из S3
|
||||
s3_keys = set()
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=S3_BUCKET):
|
||||
for obj in page.get("Contents", []):
|
||||
s3_keys.add(obj["Key"])
|
||||
|
||||
# 2. Получаем все s3_key из БД
|
||||
db_keys = set()
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT s3_key FROM attachments")
|
||||
for row in cur.fetchall():
|
||||
db_keys.add(row[0])
|
||||
|
||||
# 3. Сравниваем
|
||||
orphans = s3_keys - db_keys
|
||||
missing = db_keys - s3_keys
|
||||
|
||||
print(f"\n--- RECONCILIATION REPORT ---")
|
||||
print(f"Total objects in S3: {len(s3_keys)}")
|
||||
print(f"Total records in DB: {len(db_keys)}")
|
||||
print(f"Orphan objects (in S3, not in DB): {len(orphans)}")
|
||||
print(f"Missing objects (in DB, not in S3): {len(missing)}")
|
||||
|
||||
if orphans:
|
||||
print("\n[ORPHANS] (Safe to delete, no metadata in DB):")
|
||||
for key in orphans:
|
||||
print(f" - {key}")
|
||||
|
||||
if missing:
|
||||
print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):")
|
||||
for key in missing:
|
||||
print(f" - {key}")
|
||||
|
||||
if args.fix and orphans:
|
||||
print("\n[FIX] Deleting orphan objects from S3...")
|
||||
for key in orphans:
|
||||
s3.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
print(f" Deleted {key}")
|
||||
|
||||
if not args.fix and (orphans or missing):
|
||||
print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user