forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
169 lines
6.9 KiB
Python
169 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Upload → stop the API replica that served the upload → download via another replica.
|
|
|
|
Runs one round per replica (the stopped one is restored before the next round).
|
|
Usage: python scripts/attachment_failover_evidence.py
|
|
Writes evidence/attachment-failover.json; tokens are never printed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
import imagegen # noqa: E402
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
|
|
|
|
|
def sh(args) -> str:
|
|
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
|
|
|
|
|
|
def request(method, path, body=None, token=None, raw=None, headers=None):
|
|
h = {"Accept": "application/json"}
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
if body is not None:
|
|
raw = json.dumps(body).encode()
|
|
h["Content-Type"] = "application/json"
|
|
h.update(headers or {})
|
|
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
|
try:
|
|
resp = urllib.request.urlopen(req, timeout=10)
|
|
except urllib.error.HTTPError as err:
|
|
resp = err
|
|
return resp.code, resp.headers, resp.read()
|
|
|
|
|
|
def actor(label):
|
|
username = f"fo_{label}_{uuid.uuid4().hex[:10]}"
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
|
assert status == 201
|
|
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
|
assert status == 201
|
|
return json.loads(data)["id"], json.loads(data2)["token"]
|
|
|
|
|
|
def api_containers():
|
|
rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api",
|
|
"--filter", "label=com.docker.compose.project=mtusi2026lab4",
|
|
"--format", "{{.ID}}"]).split()
|
|
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip():
|
|
sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows
|
|
|
|
|
|
def observe(token, n=30):
|
|
return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)}
|
|
|
|
|
|
def wait_two(token, timeout=90):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
try:
|
|
if len(observe(token, 20)) >= 2:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(1)
|
|
raise RuntimeError("two replicas not observed")
|
|
|
|
|
|
def one_round(index, alice, bob, chat_id, seen_targets):
|
|
data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
for _ in range(40):
|
|
status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png",
|
|
token=alice, raw=data, headers={"Content-Type": "image/png"})
|
|
assert status == 201, status
|
|
target = h.get("X-Instance-Id")
|
|
if target not in seen_targets:
|
|
break
|
|
att = json.loads(body)
|
|
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]},
|
|
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
|
|
assert status == 201
|
|
before, ids = api_containers()
|
|
victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target)
|
|
t0 = time.monotonic()
|
|
sh(["docker", "stop", "-t", "15", victim])
|
|
stop_seconds = time.monotonic() - t0
|
|
downloads, first_ok, errors = [], None, 0
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline and len(downloads) < 20:
|
|
try:
|
|
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
|
|
except OSError:
|
|
errors += 1
|
|
continue
|
|
if status == 200:
|
|
if first_ok is None:
|
|
first_ok = time.monotonic() - t0
|
|
downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest})
|
|
else:
|
|
errors += 1
|
|
after, _ = api_containers()
|
|
meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0]
|
|
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
|
|
linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"])
|
|
result = {
|
|
"round": index,
|
|
"upload_served_by": target,
|
|
"stopped_container_state_after": after.get(target),
|
|
"docker_stop_seconds": round(stop_seconds, 2),
|
|
"first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None,
|
|
"downloads": len(downloads),
|
|
"download_instances": sorted({d["instance"] for d in downloads}),
|
|
"target_served_any_download": any(d["instance"] == target for d in downloads),
|
|
"all_sha256_match": all(d["sha256_ok"] for d in downloads),
|
|
"errors_during_window": errors,
|
|
"metadata_status": meta_status,
|
|
"message_link_preserved": linked,
|
|
"containers_before": before,
|
|
"containers_after": after,
|
|
}
|
|
sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"])
|
|
wait_two(alice)
|
|
seen_targets.add(target)
|
|
return result
|
|
|
|
|
|
def main():
|
|
alice_id, alice = actor("a")
|
|
bob_id, bob = actor("b")
|
|
status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice)
|
|
assert status == 201
|
|
chat_id = json.loads(data)["id"]
|
|
wait_two(alice)
|
|
seen = set()
|
|
rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)]
|
|
ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"]
|
|
and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds)
|
|
report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds}
|
|
out = ROOT / "evidence" / "attachment-failover.json"
|
|
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
for r in rounds:
|
|
print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after",
|
|
"first_download_ok_after_stop_s", "downloads", "download_instances",
|
|
"target_served_any_download", "all_sha256_match", "errors_during_window")}))
|
|
print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out)
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|