import os import sys import boto3 from botocore.client import Config import psycopg2 import psycopg2.extras from datetime import datetime, timezone, timedelta DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") s3_client = boto3.client( "s3", endpoint_url=S3_ENDPOINT, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, config=Config(signature_version="s3v4"), region_name="us-east-1" ) def run_reconciliation(dry_run: bool = False): print(f"Starting reconciliation (dry_run={dry_run})...") conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) threshold = datetime.now(timezone.utc) - timedelta(hours=1) with conn.cursor() as cur: cur.execute(""" SELECT id, chat_id, filename, created_at FROM attachments WHERE status = 'pending' AND created_at < %s """, (threshold,)) stale_pending = cur.fetchall() for row in stale_pending: att_id = str(row['id']) chat_id = str(row['chat_id']) filename = row['filename'] s3_key = f"attachments/{chat_id}/{att_id}/{filename}" print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.") if not dry_run: try: s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key) except Exception as e: print(f"Failed to delete S3 key {s3_key}: {e}") with conn.cursor() as cur: cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,)) conn.commit() with conn.cursor() as cur: cur.execute("SELECT id, chat_id, filename FROM attachments") all_db_attachments = cur.fetchall() valid_s3_keys = set() for row in all_db_attachments: valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}") paginator = s3_client.get_paginator('list_objects_v2') for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"): for obj in page.get('Contents', []): key = obj['Key'] last_modified = obj['LastModified'] if key not in valid_s3_keys and last_modified < threshold: print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})") if not dry_run: try: s3_client.delete_object(Bucket=S3_BUCKET, Key=key) print(f"Deleted orphan object {key} from S3.") except Exception as e: print(f"Failed to delete orphan {key}: {e}") conn.close() print("Reconciliation completed successfully.") if __name__ == "__main__": dry_run_mode = "--dry-run" in sys.argv run_reconciliation(dry_run=dry_run_mode)