Files
mtusi2026lab4/scripts/attachment_failover_evidence.py
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

169 lines
6.9 KiB
Python

#!/usr/bin/env python3
"""Upload → stop the API replica that served the upload → download via another replica.
Runs one round per replica (the stopped one is restored before the next round).
Usage: python scripts/attachment_failover_evidence.py
Writes evidence/attachment-failover.json; tokens are never printed.
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
def request(method, path, body=None, token=None, raw=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=10)
except urllib.error.HTTPError as err:
resp = err
return resp.code, resp.headers, resp.read()
def actor(label):
username = f"fo_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201
return json.loads(data)["id"], json.loads(data2)["token"]
def api_containers():
rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.project=mtusi2026lab4",
"--format", "{{.ID}}"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip():
sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows
def observe(token, n=30):
return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)}
def wait_two(token, timeout=90):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if len(observe(token, 20)) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("two replicas not observed")
def one_round(index, alice, bob, chat_id, seen_targets):
data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible
digest = hashlib.sha256(data).hexdigest()
for _ in range(40):
status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png",
token=alice, raw=data, headers={"Content-Type": "image/png"})
assert status == 201, status
target = h.get("X-Instance-Id")
if target not in seen_targets:
break
att = json.loads(body)
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]},
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
assert status == 201
before, ids = api_containers()
victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target)
t0 = time.monotonic()
sh(["docker", "stop", "-t", "15", victim])
stop_seconds = time.monotonic() - t0
downloads, first_ok, errors = [], None, 0
deadline = time.monotonic() + 30
while time.monotonic() < deadline and len(downloads) < 20:
try:
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
except OSError:
errors += 1
continue
if status == 200:
if first_ok is None:
first_ok = time.monotonic() - t0
downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest})
else:
errors += 1
after, _ = api_containers()
meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0]
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"])
result = {
"round": index,
"upload_served_by": target,
"stopped_container_state_after": after.get(target),
"docker_stop_seconds": round(stop_seconds, 2),
"first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None,
"downloads": len(downloads),
"download_instances": sorted({d["instance"] for d in downloads}),
"target_served_any_download": any(d["instance"] == target for d in downloads),
"all_sha256_match": all(d["sha256_ok"] for d in downloads),
"errors_during_window": errors,
"metadata_status": meta_status,
"message_link_preserved": linked,
"containers_before": before,
"containers_after": after,
}
sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"])
wait_two(alice)
seen_targets.add(target)
return result
def main():
alice_id, alice = actor("a")
bob_id, bob = actor("b")
status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice)
assert status == 201
chat_id = json.loads(data)["id"]
wait_two(alice)
seen = set()
rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)]
ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"]
and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds)
report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds}
out = ROOT / "evidence" / "attachment-failover.json"
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
for r in rounds:
print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after",
"first_download_ok_after_stop_s", "downloads", "download_instances",
"target_served_any_download", "all_sha256_match", "errors_during_window")}))
print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out)
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())