Files
mtusi2026lab4/research/reconcile_bench.py
T

208 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 2 (согласованность двух хранилищ): reconciler S3 <-> PostgreSQL — внедрение ошибок, dry-run/apply, стоимость обхода.
Сценарий (стенд запущен; используются синтетические данные):
1. Наращиваем число вложений тремя порциями (до ~N) и после каждой порции замеряем стоимость dry-run обхода.
2. Внедряем ошибки на ОБЕИХ сторонах: A) 20 «осиротевших» объектов в S3 (нет строки в БД);
B) у 20 вложений удаляем объект в S3 (строка есть, объекта нет); C) у 5 вложений подменяем объект на другой по размеру.
3. dry-run: находит ровно внедрённое и ничего не меняет (повтор даёт тот же отчёт);
apply с grace=5 с ПАРАЛЛЕЛЬНО идущей загрузкой 200 новых вложений: удаляются только старые осиротевшие объекты;
свежие/используемые вложения не затронуты (скачиваются, sha256 совпадает);
повторный apply — идемпотентен (изменений нет).
Результат: evidence/track2_reconcile.json
python research/reconcile_bench.py --out evidence --total 3000
"""
import argparse
import asyncio
import base64
import hashlib
import io
import json
import os
import random
import subprocess
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
def sh(*args):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True)
def compose(*args):
return sh("docker", "compose", "--env-file", ".env", *args).stdout
def reconcile(*args):
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", "--max-listed", "100", *args)
return json.loads(out[out.index("{"):])
def in_api(code):
r = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code)
if r.returncode != 0:
raise RuntimeError("сбой во внедрении ошибок: " + r.stderr[-500:])
return r.stdout.strip()
def tiny_png(seed):
img = Image.new("RGB", (40, 30), (seed % 256, (seed // 256) % 256, 90))
buf = io.BytesIO()
img.save(buf, "PNG")
return buf.getvalue()
async def register(s):
name = "rc_%s" % uuid.uuid4().hex[:12]
pw = "reconcile-test-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "r", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "reconcile", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
async def upload_many(s, token, chat_id, n, offset=0, concurrency=16):
ids, sem = [], asyncio.Semaphore(concurrency)
async def one(i):
async with sem:
data = tiny_png(offset + i)
for attempt in range(4): # повтор при разрыве keep-alive соединения (гонка закрытия на стороне сервера)
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=r%d.png" % (BASE, chat_id, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
body = await r.json(content_type=None)
if r.status == 201:
ids.append((body["id"], body["sha256"]))
break
except aiohttp.ClientError:
await asyncio.sleep(0.1)
await asyncio.gather(*(one(i) for i in range(n)))
return ids
async def check_intact(s, token, items):
ok = 0
for att_id, sha in items:
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att_id), headers={"Authorization": "Bearer " + token}) as r:
if r.status == 200 and hashlib.sha256(await r.read()).hexdigest() == sha:
ok += 1
return ok
def keys_for(ids):
out = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
"SELECT s3_key FROM attachments WHERE id IN (%s)" % ",".join("'%s'" % i for i in ids)).stdout.split()
return out
async def bind_to_messages(s, token, chat, ids):
"""Create real message references, not just ready attachment metadata."""
headers = {"Authorization": "Bearer " + token}
for start in range(0, len(ids), 10):
batch = ids[start:start + 10]
async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat),
json={"text": "reconciler keeper", "attachment_ids": batch},
headers={**headers, "Idempotency-Key": str(uuid.uuid4())}) as r:
body = await r.json()
assert r.status == 201 and body["attachment_ids"] == batch, (r.status, body)
async def message_bindings(s, token, chat):
async with s.get("%s/api/v1/chats/%s/messages?limit=100" % (BASE, chat),
headers={"Authorization": "Bearer " + token}) as r:
r.raise_for_status()
body = await r.json()
return {m["id"]: m["attachment_ids"] for m in body["items"]}
async def main(args):
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "cost_of_scan": [], "args": vars(args)}
# Docker commands block the event loop longer than the server's keep-alive.
# Do not reuse idle HTTP sockets after a scan or fault-injection command.
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60),
connector=aiohttp.TCPConnector(limit=32, force_close=True)) as s:
token, chat = await register(s)
base = reconcile("--grace", "0", "--apply") # нормализация: убираем следы прошлых прогонов
base = reconcile("--grace", "0")
assert (base["orphan_objects_count"], base["missing_objects_count"], base["size_mismatch_count"]) == (0, 0, 0), base
uploaded, step = [], args.total // 3
for part in range(3):
t = time.perf_counter()
uploaded += await upload_many(s, token, chat, step, offset=len(uploaded))
up_s = time.perf_counter() - t
r = reconcile("--grace", "900")
report["cost_of_scan"].append({"db_rows": r["db_rows"], "s3_objects": r["s3_objects"], "timing_ms": r["timing_ms"],
"upload_batch_seconds": round(up_s, 1)})
print(json.dumps(report["cost_of_scan"][-1]))
# --- внедрение ошибок ---
sample = random.sample(uploaded, 25)
missing_ids = [a for a, _ in sample[:20]]
mismatch_ids = [a for a, _ in sample[20:25]]
used_ids = [a for a, _ in random.sample([u for u in uploaded if u[0] not in set(missing_ids + mismatch_ids)], 200)]
await bind_to_messages(s, token, chat, used_ids)
bindings_before = await message_bindings(s, token, chat)
assert {a for ids in bindings_before.values() for a in ids} == set(used_ids)
orphans = ["att/%s/original" % uuid.uuid4() for _ in range(20)]
mkeys, xkeys = keys_for(missing_ids), keys_for(mismatch_ids)
code = ("import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start()\n"
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'orphan')\n"
" for k in %r: await s.delete(k)\n"
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'x'*7)\n await s.stop()\nasyncio.run(m())" % (orphans, mkeys, xkeys))
in_api(code)
report["injected"] = {"orphan_objects": 20, "missing_objects": 20, "size_mismatch": 5}
used_pairs = [u for u in uploaded if u[0] in set(used_ids)]
time.sleep(6) # осиротевшие объекты станут старше grace=5 с
# --- dry-run ---
dry1 = reconcile("--grace", "5")
dry2 = reconcile("--grace", "5")
report["dry_run"] = {k: dry1[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count", "orphans_skipped_fresh", "timing_ms", "fixed")}
report["dry_run_repeatable"] = all(dry1[k] == dry2[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count"))
report["dry_run_found_exactly_injected"] = (dry1["orphan_objects_count"], dry1["missing_objects_count"], dry1["size_mismatch_count"]) == (20, 20, 5)
# --- apply параллельно с загрузкой новых (свежих) вложений ---
fresh_task = asyncio.create_task(upload_many(s, token, chat, 200, offset=100000))
await asyncio.sleep(1.0)
loop = asyncio.get_running_loop()
applied = await loop.run_in_executor(None, lambda: reconcile("--grace", "5", "--apply"))
fresh = await fresh_task
report["apply"] = {k: applied[k] for k in ("fixed", "orphans_skipped_fresh", "timing_ms", "orphan_objects_count", "missing_objects_count", "size_mismatch_count")}
report["fresh_uploads_during_apply"] = {"uploaded": len(fresh), "intact": await check_intact(s, token, fresh)}
report["in_use_attachments_intact"] = "%d/%d" % (await check_intact(s, token, used_pairs), len(used_pairs))
bindings_after = await message_bindings(s, token, chat)
report["message_bindings"] = {"messages": len(bindings_before), "attachments": len(used_ids),
"unchanged": bindings_after == bindings_before}
again = reconcile("--grace", "5", "--apply")
report["second_apply_idempotent"] = {"fixed": again["fixed"], "orphan_objects_count": again["orphan_objects_count"],
"missing_objects_count": again["missing_objects_count"]}
failed = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
"SELECT error_code || '=' || count(*) FROM attachments WHERE status = 'failed' AND chat_id = '%s' GROUP BY error_code ORDER BY 1" % chat).stdout.split()
report["marked_failed"] = failed
with open(os.path.join(args.out, "track2_reconcile.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
print(json.dumps({k: v for k, v in report.items() if k != "args"}, ensure_ascii=False, indent=2))
assert len(uploaded) == 3 * (args.total // 3), "not all requested uploads succeeded"
assert report["dry_run_repeatable"] and report["dry_run_found_exactly_injected"], report["dry_run"]
assert dry1["fixed"] == dry2["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}
assert applied["fixed"] == {"deleted_orphans": 20, "marked_failed": 25}, applied
assert report["fresh_uploads_during_apply"] == {"uploaded": 200, "intact": 200}
assert report["in_use_attachments_intact"] == "200/200" and report["message_bindings"]["unchanged"]
assert again["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}, again
assert (again["orphan_objects_count"], again["missing_objects_count"], again["size_mismatch_count"]) == (0, 0, 0)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--total", type=int, default=3000)
asyncio.run(main(ap.parse_args()))