forked from mtusi2026/mtusi2026lab4
208 lines
12 KiB
Python
208 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Трек 2 (согласованность двух хранилищ): reconciler S3 <-> PostgreSQL — внедрение ошибок, dry-run/apply, стоимость обхода.
|
||
|
||
Сценарий (стенд запущен; используются синтетические данные):
|
||
1. Наращиваем число вложений тремя порциями (до ~N) и после каждой порции замеряем стоимость dry-run обхода.
|
||
2. Внедряем ошибки на ОБЕИХ сторонах: A) 20 «осиротевших» объектов в S3 (нет строки в БД);
|
||
B) у 20 вложений удаляем объект в S3 (строка есть, объекта нет); C) у 5 вложений подменяем объект на другой по размеру.
|
||
3. dry-run: находит ровно внедрённое и ничего не меняет (повтор даёт тот же отчёт);
|
||
apply с grace=5 с ПАРАЛЛЕЛЬНО идущей загрузкой 200 новых вложений: удаляются только старые осиротевшие объекты;
|
||
свежие/используемые вложения не затронуты (скачиваются, sha256 совпадает);
|
||
повторный apply — идемпотентен (изменений нет).
|
||
Результат: evidence/track2_reconcile.json
|
||
python research/reconcile_bench.py --out evidence --total 3000
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import base64
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import random
|
||
import subprocess
|
||
import time
|
||
import uuid
|
||
|
||
import aiohttp
|
||
from PIL import Image
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
BASE = "http://127.0.0.1:8080"
|
||
|
||
|
||
def sh(*args):
|
||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True)
|
||
|
||
|
||
def compose(*args):
|
||
return sh("docker", "compose", "--env-file", ".env", *args).stdout
|
||
|
||
|
||
def reconcile(*args):
|
||
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", "--max-listed", "100", *args)
|
||
return json.loads(out[out.index("{"):])
|
||
|
||
|
||
def in_api(code):
|
||
r = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code)
|
||
if r.returncode != 0:
|
||
raise RuntimeError("сбой во внедрении ошибок: " + r.stderr[-500:])
|
||
return r.stdout.strip()
|
||
|
||
|
||
def tiny_png(seed):
|
||
img = Image.new("RGB", (40, 30), (seed % 256, (seed // 256) % 256, 90))
|
||
buf = io.BytesIO()
|
||
img.save(buf, "PNG")
|
||
return buf.getvalue()
|
||
|
||
|
||
async def register(s):
|
||
name = "rc_%s" % uuid.uuid4().hex[:12]
|
||
pw = "reconcile-test-" + uuid.uuid4().hex[:6]
|
||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "r", "password": pw}) as r:
|
||
user = await r.json()
|
||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||
token = (await r.json())["token"]
|
||
async with s.post(BASE + "/api/v1/chats", json={"title": "reconcile", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
|
||
chat = await r.json()
|
||
return token, chat["id"]
|
||
|
||
|
||
async def upload_many(s, token, chat_id, n, offset=0, concurrency=16):
|
||
ids, sem = [], asyncio.Semaphore(concurrency)
|
||
|
||
async def one(i):
|
||
async with sem:
|
||
data = tiny_png(offset + i)
|
||
for attempt in range(4): # повтор при разрыве keep-alive соединения (гонка закрытия на стороне сервера)
|
||
try:
|
||
async with s.post("%s/api/v1/chats/%s/attachments?filename=r%d.png" % (BASE, chat_id, i), data=data,
|
||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
|
||
body = await r.json(content_type=None)
|
||
if r.status == 201:
|
||
ids.append((body["id"], body["sha256"]))
|
||
break
|
||
except aiohttp.ClientError:
|
||
await asyncio.sleep(0.1)
|
||
await asyncio.gather(*(one(i) for i in range(n)))
|
||
return ids
|
||
|
||
|
||
async def check_intact(s, token, items):
|
||
ok = 0
|
||
for att_id, sha in items:
|
||
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att_id), headers={"Authorization": "Bearer " + token}) as r:
|
||
if r.status == 200 and hashlib.sha256(await r.read()).hexdigest() == sha:
|
||
ok += 1
|
||
return ok
|
||
|
||
|
||
def keys_for(ids):
|
||
out = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
|
||
"SELECT s3_key FROM attachments WHERE id IN (%s)" % ",".join("'%s'" % i for i in ids)).stdout.split()
|
||
return out
|
||
|
||
|
||
async def bind_to_messages(s, token, chat, ids):
|
||
"""Create real message references, not just ready attachment metadata."""
|
||
headers = {"Authorization": "Bearer " + token}
|
||
for start in range(0, len(ids), 10):
|
||
batch = ids[start:start + 10]
|
||
async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat),
|
||
json={"text": "reconciler keeper", "attachment_ids": batch},
|
||
headers={**headers, "Idempotency-Key": str(uuid.uuid4())}) as r:
|
||
body = await r.json()
|
||
assert r.status == 201 and body["attachment_ids"] == batch, (r.status, body)
|
||
|
||
|
||
async def message_bindings(s, token, chat):
|
||
async with s.get("%s/api/v1/chats/%s/messages?limit=100" % (BASE, chat),
|
||
headers={"Authorization": "Bearer " + token}) as r:
|
||
r.raise_for_status()
|
||
body = await r.json()
|
||
return {m["id"]: m["attachment_ids"] for m in body["items"]}
|
||
|
||
|
||
async def main(args):
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "cost_of_scan": [], "args": vars(args)}
|
||
# Docker commands block the event loop longer than the server's keep-alive.
|
||
# Do not reuse idle HTTP sockets after a scan or fault-injection command.
|
||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60),
|
||
connector=aiohttp.TCPConnector(limit=32, force_close=True)) as s:
|
||
token, chat = await register(s)
|
||
base = reconcile("--grace", "0", "--apply") # нормализация: убираем следы прошлых прогонов
|
||
base = reconcile("--grace", "0")
|
||
assert (base["orphan_objects_count"], base["missing_objects_count"], base["size_mismatch_count"]) == (0, 0, 0), base
|
||
uploaded, step = [], args.total // 3
|
||
for part in range(3):
|
||
t = time.perf_counter()
|
||
uploaded += await upload_many(s, token, chat, step, offset=len(uploaded))
|
||
up_s = time.perf_counter() - t
|
||
r = reconcile("--grace", "900")
|
||
report["cost_of_scan"].append({"db_rows": r["db_rows"], "s3_objects": r["s3_objects"], "timing_ms": r["timing_ms"],
|
||
"upload_batch_seconds": round(up_s, 1)})
|
||
print(json.dumps(report["cost_of_scan"][-1]))
|
||
# --- внедрение ошибок ---
|
||
sample = random.sample(uploaded, 25)
|
||
missing_ids = [a for a, _ in sample[:20]]
|
||
mismatch_ids = [a for a, _ in sample[20:25]]
|
||
used_ids = [a for a, _ in random.sample([u for u in uploaded if u[0] not in set(missing_ids + mismatch_ids)], 200)]
|
||
await bind_to_messages(s, token, chat, used_ids)
|
||
bindings_before = await message_bindings(s, token, chat)
|
||
assert {a for ids in bindings_before.values() for a in ids} == set(used_ids)
|
||
orphans = ["att/%s/original" % uuid.uuid4() for _ in range(20)]
|
||
mkeys, xkeys = keys_for(missing_ids), keys_for(mismatch_ids)
|
||
code = ("import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start()\n"
|
||
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'orphan')\n"
|
||
" for k in %r: await s.delete(k)\n"
|
||
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'x'*7)\n await s.stop()\nasyncio.run(m())" % (orphans, mkeys, xkeys))
|
||
in_api(code)
|
||
report["injected"] = {"orphan_objects": 20, "missing_objects": 20, "size_mismatch": 5}
|
||
used_pairs = [u for u in uploaded if u[0] in set(used_ids)]
|
||
time.sleep(6) # осиротевшие объекты станут старше grace=5 с
|
||
# --- dry-run ---
|
||
dry1 = reconcile("--grace", "5")
|
||
dry2 = reconcile("--grace", "5")
|
||
report["dry_run"] = {k: dry1[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count", "orphans_skipped_fresh", "timing_ms", "fixed")}
|
||
report["dry_run_repeatable"] = all(dry1[k] == dry2[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count"))
|
||
report["dry_run_found_exactly_injected"] = (dry1["orphan_objects_count"], dry1["missing_objects_count"], dry1["size_mismatch_count"]) == (20, 20, 5)
|
||
# --- apply параллельно с загрузкой новых (свежих) вложений ---
|
||
fresh_task = asyncio.create_task(upload_many(s, token, chat, 200, offset=100000))
|
||
await asyncio.sleep(1.0)
|
||
loop = asyncio.get_running_loop()
|
||
applied = await loop.run_in_executor(None, lambda: reconcile("--grace", "5", "--apply"))
|
||
fresh = await fresh_task
|
||
report["apply"] = {k: applied[k] for k in ("fixed", "orphans_skipped_fresh", "timing_ms", "orphan_objects_count", "missing_objects_count", "size_mismatch_count")}
|
||
report["fresh_uploads_during_apply"] = {"uploaded": len(fresh), "intact": await check_intact(s, token, fresh)}
|
||
report["in_use_attachments_intact"] = "%d/%d" % (await check_intact(s, token, used_pairs), len(used_pairs))
|
||
bindings_after = await message_bindings(s, token, chat)
|
||
report["message_bindings"] = {"messages": len(bindings_before), "attachments": len(used_ids),
|
||
"unchanged": bindings_after == bindings_before}
|
||
again = reconcile("--grace", "5", "--apply")
|
||
report["second_apply_idempotent"] = {"fixed": again["fixed"], "orphan_objects_count": again["orphan_objects_count"],
|
||
"missing_objects_count": again["missing_objects_count"]}
|
||
failed = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
|
||
"SELECT error_code || '=' || count(*) FROM attachments WHERE status = 'failed' AND chat_id = '%s' GROUP BY error_code ORDER BY 1" % chat).stdout.split()
|
||
report["marked_failed"] = failed
|
||
with open(os.path.join(args.out, "track2_reconcile.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
print(json.dumps({k: v for k, v in report.items() if k != "args"}, ensure_ascii=False, indent=2))
|
||
assert len(uploaded) == 3 * (args.total // 3), "not all requested uploads succeeded"
|
||
assert report["dry_run_repeatable"] and report["dry_run_found_exactly_injected"], report["dry_run"]
|
||
assert dry1["fixed"] == dry2["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}
|
||
assert applied["fixed"] == {"deleted_orphans": 20, "marked_failed": 25}, applied
|
||
assert report["fresh_uploads_during_apply"] == {"uploaded": 200, "intact": 200}
|
||
assert report["in_use_attachments_intact"] == "200/200" and report["message_bindings"]["unchanged"]
|
||
assert again["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}, again
|
||
assert (again["orphan_objects_count"], again["missing_objects_count"], again["size_mismatch_count"]) == (0, 0, 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--out", default="evidence")
|
||
ap.add_argument("--total", type=int, default=3000)
|
||
asyncio.run(main(ap.parse_args()))
|