Files
mtusi2026lab4/scripts/reconcile_experiment.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

143 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Fault injection for the DB<->S3 reconciler. Run inside an api container:
docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]"""
import argparse
import asyncio
import io
import json
import sys
import time
import uuid
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
ZERO = "0" * 64
checks = []
def check(name, condition):
checks.append((name, bool(condition)))
print(("ok " if condition else "FAIL ") + name)
async def main():
parser = argparse.ArgumentParser()
parser.add_argument("--bulk", type=int, default=1000)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png")
exists = lambda key: run_in_threadpool(s3.exists, key)
baseline = await reconcile.run(db, s3, 900, dry_run=True)
print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True))
async with db.connection() as conn:
suffix = uuid.uuid4().hex[:10]
user = (await (await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"]
chat = (await (await conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"]
await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user))
async def row(status, age_s, with_object, linked=False):
aid = str(uuid.uuid4())
key = "attachments/" + aid
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s,"
" now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s))
if with_object:
await put(key)
if linked:
mid = (await (await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"]
await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid))
return aid, key
async def status_of(aid):
r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone()
return (r["status"], r["error_code"]) if r else None
old = 7200
a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young)
b_id, b_key = await row("uploading", old, False) # crash before PUT
c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive
e_id, e_key = await row("ready", old, False) # object lost
f_id, f_key = await row("ready", old, True, linked=True) # healthy, used
g_id, g_key = await row("ready", old, False, linked=True) # object lost, used
h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key
await put(h_key)
i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked
d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written
await put(d_key)
print("== dry-run, grace=900 ==")
r = await reconcile.run(db, s3, 900, dry_run=True)
print(json.dumps(r, sort_keys=True))
check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)",
r["stale_uploading"] - baseline["stale_uploading"] == 1
and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2)
check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2)
check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None)
and await status_of(e_id) == ("ready", None))
print("== apply, grace=900 ==")
r = await reconcile.run(db, s3, 900, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key))
check("stale row without object removed", await status_of(b_id) is None)
check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key))
check("fresh orphan object kept inside grace", await exists(d_key))
check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key))
check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None))
check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing"))
check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing"))
print("== apply after grace expired (grace=2 after 3s) ==")
await asyncio.sleep(3)
r = await reconcile.run(db, s3, 2, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("orphan object deleted after grace", not await exists(d_key))
check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key))
check("old row + object collected once the object is older than grace (a, h)",
await status_of(a_id) is None and not await exists(a_key)
and await status_of(h_id) is None and not await exists(h_key))
check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
print("== second apply is idempotent ==")
r = await reconcile.run(db, s3, 2, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0)
print("== cost of a full walk, %d ready attachments ==" % args.bulk)
keys = []
for _ in range(args.bulk):
aid, key = await row("ready", old, True)
keys.append(aid)
r = await reconcile.run(db, s3, 900, dry_run=True)
print(json.dumps(r, sort_keys=True))
check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0)
for aid in keys + [e_id, f_id, g_id, h_id, i_id]:
await run_in_threadpool(s3.delete, "attachments/" + aid)
await conn.execute("DELETE FROM chats WHERE id = %s", (chat,))
await conn.execute("DELETE FROM users WHERE id = %s", (user,))
await db.close()
failed = [n for n, ok in checks if not ok]
print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks)))
sys.exit(1 if failed else 0)
asyncio.run(main())