Files
mtusi2026lab5/scripts/reconcile.py
T
2026-11-16 14:44:03 +03:00

81 lines
3.1 KiB
Python

import os
import sys
import boto3
from botocore.client import Config
import psycopg2
import psycopg2.extras
from datetime import datetime, timezone, timedelta
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
def run_reconciliation(dry_run: bool = False):
print(f"Starting reconciliation (dry_run={dry_run})...")
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
with conn.cursor() as cur:
cur.execute("""
SELECT id, chat_id, filename, created_at FROM attachments
WHERE status = 'pending' AND created_at < %s
""", (threshold,))
stale_pending = cur.fetchall()
for row in stale_pending:
att_id = str(row['id'])
chat_id = str(row['chat_id'])
filename = row['filename']
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
if not dry_run:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
except Exception as e:
print(f"Failed to delete S3 key {s3_key}: {e}")
with conn.cursor() as cur:
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
conn.commit()
with conn.cursor() as cur:
cur.execute("SELECT id, chat_id, filename FROM attachments")
all_db_attachments = cur.fetchall()
valid_s3_keys = set()
for row in all_db_attachments:
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
paginator = s3_client.get_paginator('list_objects_v2')
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
for obj in page.get('Contents', []):
key = obj['Key']
last_modified = obj['LastModified']
if key not in valid_s3_keys and last_modified < threshold:
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
if not dry_run:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
print(f"Deleted orphan object {key} from S3.")
except Exception as e:
print(f"Failed to delete orphan {key}: {e}")
conn.close()
print("Reconciliation completed successfully.")
if __name__ == "__main__":
dry_run_mode = "--dry-run" in sys.argv
run_reconciliation(dry_run=dry_run_mode)