forked from mtusi2026/mtusi2026lab4
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
143 lines
7.6 KiB
Python
143 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Fault injection for the DB<->S3 reconciler. Run inside an api container:
|
|
docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]"""
|
|
import argparse
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, "/app")
|
|
|
|
from starlette.concurrency import run_in_threadpool # noqa: E402
|
|
|
|
from app import config, reconcile # noqa: E402
|
|
from app.db import Database # noqa: E402
|
|
from app.s3 import S3Storage # noqa: E402
|
|
|
|
ZERO = "0" * 64
|
|
checks = []
|
|
|
|
|
|
def check(name, condition):
|
|
checks.append((name, bool(condition)))
|
|
print(("ok " if condition else "FAIL ") + name)
|
|
|
|
|
|
async def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--bulk", type=int, default=1000)
|
|
args = parser.parse_args()
|
|
db, s3 = Database(), S3Storage()
|
|
await db.open()
|
|
await run_in_threadpool(s3.open)
|
|
put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png")
|
|
exists = lambda key: run_in_threadpool(s3.exists, key)
|
|
|
|
baseline = await reconcile.run(db, s3, 900, dry_run=True)
|
|
print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True))
|
|
|
|
async with db.connection() as conn:
|
|
suffix = uuid.uuid4().hex[:10]
|
|
user = (await (await conn.execute(
|
|
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
|
" VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"]
|
|
chat = (await (await conn.execute(
|
|
"INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"]
|
|
await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user))
|
|
|
|
async def row(status, age_s, with_object, linked=False):
|
|
aid = str(uuid.uuid4())
|
|
key = "attachments/" + aid
|
|
await conn.execute(
|
|
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
|
" status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s,"
|
|
" now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s))
|
|
if with_object:
|
|
await put(key)
|
|
if linked:
|
|
mid = (await (await conn.execute(
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
|
" VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"]
|
|
await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid))
|
|
return aid, key
|
|
|
|
async def status_of(aid):
|
|
r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone()
|
|
return (r["status"], r["error_code"]) if r else None
|
|
|
|
old = 7200
|
|
a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young)
|
|
b_id, b_key = await row("uploading", old, False) # crash before PUT
|
|
c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive
|
|
e_id, e_key = await row("ready", old, False) # object lost
|
|
f_id, f_key = await row("ready", old, True, linked=True) # healthy, used
|
|
g_id, g_key = await row("ready", old, False, linked=True) # object lost, used
|
|
h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key
|
|
await put(h_key)
|
|
i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked
|
|
d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written
|
|
await put(d_key)
|
|
|
|
print("== dry-run, grace=900 ==")
|
|
r = await reconcile.run(db, s3, 900, dry_run=True)
|
|
print(json.dumps(r, sort_keys=True))
|
|
check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)",
|
|
r["stale_uploading"] - baseline["stale_uploading"] == 1
|
|
and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2)
|
|
check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2)
|
|
check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None)
|
|
and await status_of(e_id) == ("ready", None))
|
|
|
|
print("== apply, grace=900 ==")
|
|
r = await reconcile.run(db, s3, 900, dry_run=False)
|
|
print(json.dumps(r, sort_keys=True))
|
|
check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key))
|
|
check("stale row without object removed", await status_of(b_id) is None)
|
|
check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key))
|
|
check("fresh orphan object kept inside grace", await exists(d_key))
|
|
check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key))
|
|
check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None))
|
|
check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing"))
|
|
check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
|
check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing"))
|
|
|
|
print("== apply after grace expired (grace=2 after 3s) ==")
|
|
await asyncio.sleep(3)
|
|
r = await reconcile.run(db, s3, 2, dry_run=False)
|
|
print(json.dumps(r, sort_keys=True))
|
|
check("orphan object deleted after grace", not await exists(d_key))
|
|
check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key))
|
|
check("old row + object collected once the object is older than grace (a, h)",
|
|
await status_of(a_id) is None and not await exists(a_key)
|
|
and await status_of(h_id) is None and not await exists(h_key))
|
|
check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
|
|
|
print("== second apply is idempotent ==")
|
|
r = await reconcile.run(db, s3, 2, dry_run=False)
|
|
print(json.dumps(r, sort_keys=True))
|
|
check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0)
|
|
|
|
print("== cost of a full walk, %d ready attachments ==" % args.bulk)
|
|
keys = []
|
|
for _ in range(args.bulk):
|
|
aid, key = await row("ready", old, True)
|
|
keys.append(aid)
|
|
r = await reconcile.run(db, s3, 900, dry_run=True)
|
|
print(json.dumps(r, sort_keys=True))
|
|
check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0)
|
|
|
|
for aid in keys + [e_id, f_id, g_id, h_id, i_id]:
|
|
await run_in_threadpool(s3.delete, "attachments/" + aid)
|
|
await conn.execute("DELETE FROM chats WHERE id = %s", (chat,))
|
|
await conn.execute("DELETE FROM users WHERE id = %s", (user,))
|
|
await db.close()
|
|
failed = [n for n, ok in checks if not ok]
|
|
print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks)))
|
|
sys.exit(1 if failed else 0)
|
|
|
|
|
|
asyncio.run(main())
|