"""Small local experiment for the two-storage reconciler.""" import json import os import subprocess import time import uuid from pathlib import Path import boto3 import psycopg from botocore.config import Config def main(): bucket = os.getenv('S3_BUCKET', 'messenger-attachments') s3 = boto3.client( 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://minio:9000'), region_name=os.getenv('S3_REGION', 'us-east-1'), aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'messenger-api'), aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123'), config=Config(connect_timeout=2, read_timeout=5, retries={'max_attempts': 1}), ) old_key = 'reconcile-demo/old-' + uuid.uuid4().hex fresh_key = 'reconcile-demo/fresh-' + uuid.uuid4().hex s3.put_object(Bucket=bucket, Key=old_key, Body=b'old orphan') time.sleep(6) s3.put_object(Bucket=bucket, Key=fresh_key, Body=b'fresh orphan') missing_id = uuid.uuid4() with psycopg.connect(os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')) as connection: owner, chat = connection.execute('SELECT user_id, chat_id FROM chat_members LIMIT 1').fetchone() connection.execute( 'INSERT INTO attachments ' '(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,status,error_code,created_at,updated_at) ' "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),now())", (missing_id, chat, owner, 'missing.png', 'image/png', 10, '0' * 64, 'attachments/missing-' + str(missing_id), 'ready', None), ) connection.commit() root = Path(__file__).resolve().parent dry = subprocess.check_output( ['python', str(root / 'reconcile_s3.py'), '--grace-seconds', '5'], text=True, ) applied = subprocess.check_output( ['python', str(root / 'reconcile_s3.py'), '--apply', '--grace-seconds', '5'], text=True, ) applied_data = json.loads(applied) fresh_exists = True try: s3.head_object(Bucket=bucket, Key=fresh_key) except Exception: fresh_exists = False s3.delete_object(Bucket=bucket, Key=fresh_key) with psycopg.connect(os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')) as connection: missing_status = connection.execute('SELECT status,error_code FROM attachments WHERE id=%s', (missing_id,)).fetchone() connection.execute('DELETE FROM attachments WHERE id=%s', (missing_id,)) connection.commit() print(json.dumps({ 'old_orphan_removed': old_key in applied_data['deleted_orphans'], 'fresh_orphan_preserved': fresh_exists, 'missing_marked_failed': list(missing_status) == ['failed', 'object_missing'], 'dry_run': dry.strip(), 'apply': applied.strip(), }, ensure_ascii=False, indent=2)) if __name__ == '__main__': main()