Files
mtusi2026lab4/student-tests/consistency_faults.py
T
zemiles c5a07dc253 ЛР 4 — БПИ2303 — Лазарев — на 4
Перенос lab3 и вложения в приватном S3 (SeaweedFS 4.50).

- POST /chats/{id}/attachments: потоковый приём PNG/JPEG в безымянный
  временный файл, SHA-256 на лету, лимиты 10 MiB / 40 Мпикс до чтения
  и декодирования, проверка MIME по содержимому, валидация filename.
- Порядок записи pending -> S3 PUT -> ready с компенсацией; janitor
  сверяет БД и bucket (dry-run, grace period, advisory lock).
- GET /attachments/{id} и /content только участникам чата, без redirect;
  attachment_ids в сообщениях с проверкой принадлежности чату.
- Отдельные S3-ключи api/janitor/admin с ограниченными правами,
  идемпотентный bootstrap bucket, служебные порты SeaweedFS закрыты.
- Ограничение параллельных S3 PUT и MALLOC_ARENA_MAX: пик RSS API
  178-189 -> 95-102 MiB; лимиты памяти и restart для SeaweedFS после OOM.
- Тесты уровня 4 (негативные, failover вложений, отказы S3/БД, память)
  и доказательства в evidence/, заполнен REPORT.md.
2026-11-07 01:44:58 +03:00

180 lines
8.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
FAULT_INJECTION=1 docker compose --env-file .env up -d --wait api
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
docker compose --env-file .env up -d --wait api # вернуть FAULT_INJECTION=0
Сценарии (каждый — реальный HTTP upload через LB):
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
и pending-строку -> ничего не остаётся;
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
crash-after-pending процесс умирает до PUT -> pending без объекта;
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
orphan pending-строку удаляем вручную -> объект без строки;
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
должно остаться доступным побайтно.
"""
import argparse
import hashlib
import json
import socket
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
log = []
def sh(*cmd, timeout=120):
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
if r.returncode != 0:
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
return r.stdout
def psql(sql):
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
def janitor(grace, dry_run):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
out = sh(*cmd)
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
return {k: report.get(k) for k in keep}
def step(name, **data):
entry = dict(step=name, t=round(time.time(), 1), **data)
log.append(entry)
print(json.dumps(entry, ensure_ascii=False), flush=True)
return entry
def wait_api_healthy():
deadline = time.time() + 90
while time.time() < deadline:
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
if states and all(s == "healthy" for s in states):
time.sleep(3) # HAProxy rise
return
time.sleep(1)
raise SystemExit("api did not become healthy")
def upload(c, actor, chat, fault=None, expected=201):
h = {"Content-Type": "image/png"}
if fault:
h["X-Fault-Inject"] = fault
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
return r["status"], r["json"]
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
return type(exc).__name__, str(exc)[:120]
def interrupted_upload(base, token, chat_id):
host, port = base.split("//")[1].split(":")
s = socket.create_connection((host, int(port)), timeout=10)
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
s.close() # клиент «пропал» на середине
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--grace-seconds", type=int, default=20)
ap.add_argument("--output")
args = ap.parse_args()
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
started = psql("SELECT now()")
base = janitor(3600, True)
step("baseline janitor dry-run", report=base)
_, control = upload(c, alice, chat)
_, victim = upload(c, alice, chat)
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
after_comp = janitor(3600, True)
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
st, body = upload(c, alice, chat, fault, expected=[502, 503])
step(fault, client_saw=st)
wait_api_healthy()
interrupted_upload(args.base_url, alice["token"], chat["id"])
time.sleep(2)
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
step("pending rows left by crashes", pending=len(pending))
check(len(pending) == 3, "expected 3 pending rows from crashes")
orphan_id = pending[0].split()[0]
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
step("orphan injected: pending row deleted, object kept", row=orphan_id)
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
"client('api').remove_object(BUCKET, %r)" % victim_key)
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
step("missing injected: victim object deleted in S3", download_status=dl["status"])
fresh_dry = janitor(3600, True)
fresh_run = janitor(3600, False)
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
time.sleep(args.grace_seconds + 2)
dry = janitor(args.grace_seconds, True)
step("after grace: dry-run", report=dry)
real = janitor(args.grace_seconds, False)
step("after grace: real run", report=real)
again = janitor(args.grace_seconds, False)
step("after grace: repeated run (idempotent)", report=again)
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
and dry["deleted_objects"] == 0
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
and control_ok and int(still) == 0)
print(json.dumps({"pass": ok}))
if args.output:
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())