forked from mtusi2026/mtusi2026lab6
81 lines
3.1 KiB
Python
81 lines
3.1 KiB
Python
import os
|
|
import sys
|
|
import boto3
|
|
from botocore.client import Config
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
|
|
|
s3_client = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=S3_ACCESS_KEY,
|
|
aws_secret_access_key=S3_SECRET_KEY,
|
|
config=Config(signature_version="s3v4"),
|
|
region_name="us-east-1"
|
|
)
|
|
|
|
def run_reconciliation(dry_run: bool = False):
|
|
print(f"Starting reconciliation (dry_run={dry_run})...")
|
|
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
|
|
|
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT id, chat_id, filename, created_at FROM attachments
|
|
WHERE status = 'pending' AND created_at < %s
|
|
""", (threshold,))
|
|
stale_pending = cur.fetchall()
|
|
|
|
for row in stale_pending:
|
|
att_id = str(row['id'])
|
|
chat_id = str(row['chat_id'])
|
|
filename = row['filename']
|
|
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
|
|
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
|
|
|
|
if not dry_run:
|
|
try:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
|
|
except Exception as e:
|
|
print(f"Failed to delete S3 key {s3_key}: {e}")
|
|
with conn.cursor() as cur:
|
|
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
|
|
conn.commit()
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT id, chat_id, filename FROM attachments")
|
|
all_db_attachments = cur.fetchall()
|
|
|
|
valid_s3_keys = set()
|
|
for row in all_db_attachments:
|
|
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
|
|
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
|
|
for obj in page.get('Contents', []):
|
|
key = obj['Key']
|
|
last_modified = obj['LastModified']
|
|
|
|
if key not in valid_s3_keys and last_modified < threshold:
|
|
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
|
|
if not dry_run:
|
|
try:
|
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
|
|
print(f"Deleted orphan object {key} from S3.")
|
|
except Exception as e:
|
|
print(f"Failed to delete orphan {key}: {e}")
|
|
|
|
conn.close()
|
|
print("Reconciliation completed successfully.")
|
|
|
|
if __name__ == "__main__":
|
|
dry_run_mode = "--dry-run" in sys.argv
|
|
run_reconciliation(dry_run=dry_run_mode) |