#!/usr/bin/env python3 """Fault injection for the DB<->S3 reconciler. Run inside an api container: docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]""" import argparse import asyncio import io import json import sys import time import uuid sys.path.insert(0, "/app") from starlette.concurrency import run_in_threadpool # noqa: E402 from app import config, reconcile # noqa: E402 from app.db import Database # noqa: E402 from app.s3 import S3Storage # noqa: E402 ZERO = "0" * 64 checks = [] def check(name, condition): checks.append((name, bool(condition))) print(("ok " if condition else "FAIL ") + name) async def main(): parser = argparse.ArgumentParser() parser.add_argument("--bulk", type=int, default=1000) args = parser.parse_args() db, s3 = Database(), S3Storage() await db.open() await run_in_threadpool(s3.open) put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png") exists = lambda key: run_in_threadpool(s3.exists, key) baseline = await reconcile.run(db, s3, 900, dry_run=True) print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True)) async with db.connection() as conn: suffix = uuid.uuid4().hex[:10] user = (await (await conn.execute( "INSERT INTO users (id, username, display_name, password_hash, created_at)" " VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"] chat = (await (await conn.execute( "INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"] await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user)) async def row(status, age_s, with_object, linked=False): aid = str(uuid.uuid4()) key = "attachments/" + aid await conn.execute( "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," " status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s," " now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s)) if with_object: await put(key) if linked: mid = (await (await conn.execute( "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" " VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"] await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid)) return aid, key async def status_of(aid): r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone() return (r["status"], r["error_code"]) if r else None old = 7200 a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young) b_id, b_key = await row("uploading", old, False) # crash before PUT c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive e_id, e_key = await row("ready", old, False) # object lost f_id, f_key = await row("ready", old, True, linked=True) # healthy, used g_id, g_key = await row("ready", old, False, linked=True) # object lost, used h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key await put(h_key) i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written await put(d_key) print("== dry-run, grace=900 ==") r = await reconcile.run(db, s3, 900, dry_run=True) print(json.dumps(r, sort_keys=True)) check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)", r["stale_uploading"] - baseline["stale_uploading"] == 1 and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2) check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2) check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None) and await status_of(e_id) == ("ready", None)) print("== apply, grace=900 ==") r = await reconcile.run(db, s3, 900, dry_run=False) print(json.dumps(r, sort_keys=True)) check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key)) check("stale row without object removed", await status_of(b_id) is None) check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key)) check("fresh orphan object kept inside grace", await exists(d_key)) check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key)) check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None)) check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing")) check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key)) check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing")) print("== apply after grace expired (grace=2 after 3s) ==") await asyncio.sleep(3) r = await reconcile.run(db, s3, 2, dry_run=False) print(json.dumps(r, sort_keys=True)) check("orphan object deleted after grace", not await exists(d_key)) check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key)) check("old row + object collected once the object is older than grace (a, h)", await status_of(a_id) is None and not await exists(a_key) and await status_of(h_id) is None and not await exists(h_key)) check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key)) print("== second apply is idempotent ==") r = await reconcile.run(db, s3, 2, dry_run=False) print(json.dumps(r, sort_keys=True)) check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0) print("== cost of a full walk, %d ready attachments ==" % args.bulk) keys = [] for _ in range(args.bulk): aid, key = await row("ready", old, True) keys.append(aid) r = await reconcile.run(db, s3, 900, dry_run=True) print(json.dumps(r, sort_keys=True)) check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0) for aid in keys + [e_id, f_id, g_id, h_id, i_id]: await run_in_threadpool(s3.delete, "attachments/" + aid) await conn.execute("DELETE FROM chats WHERE id = %s", (chat,)) await conn.execute("DELETE FROM users WHERE id = %s", (user,)) await db.close() failed = [n for n, ok in checks if not ok] print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks))) sys.exit(1 if failed else 0) asyncio.run(main())