grade4
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
import os
|
||||
import sys
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
def run_reconciliation(dry_run: bool = False):
|
||||
print(f"Starting reconciliation (dry_run={dry_run})...")
|
||||
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, chat_id, filename, created_at FROM attachments
|
||||
WHERE status = 'pending' AND created_at < %s
|
||||
""", (threshold,))
|
||||
stale_pending = cur.fetchall()
|
||||
|
||||
for row in stale_pending:
|
||||
att_id = str(row['id'])
|
||||
chat_id = str(row['chat_id'])
|
||||
filename = row['filename']
|
||||
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
|
||||
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||
except Exception as e:
|
||||
print(f"Failed to delete S3 key {s3_key}: {e}")
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
|
||||
conn.commit()
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, chat_id, filename FROM attachments")
|
||||
all_db_attachments = cur.fetchall()
|
||||
|
||||
valid_s3_keys = set()
|
||||
for row in all_db_attachments:
|
||||
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
|
||||
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
last_modified = obj['LastModified']
|
||||
|
||||
if key not in valid_s3_keys and last_modified < threshold:
|
||||
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
|
||||
if not dry_run:
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
print(f"Deleted orphan object {key} from S3.")
|
||||
except Exception as e:
|
||||
print(f"Failed to delete orphan {key}: {e}")
|
||||
|
||||
conn.close()
|
||||
print("Reconciliation completed successfully.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
dry_run_mode = "--dry-run" in sys.argv
|
||||
run_reconciliation(dry_run=dry_run_mode)
|
||||
Reference in New Issue
Block a user