Files
mtusi2026lab4/scripts/seq_failover_evidence.py
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

172 lines
5.2 KiB
Python

#!/usr/bin/env python3
"""Grade-4 evidence: stop each observed API replica in turn; restore between steps."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
def call(method, path, body=None, token=None, headers=None, expected=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
raw = resp.read()
status = resp.code
if expected is not None and status != expected:
raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}")
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": status, "headers": resp.headers, "json": js}
def header(resp, name: str) -> str:
return resp["headers"].get(name) or resp["headers"].get(name.lower()) or ""
def restore_api(target: str):
replicas = os.environ.get("API_REPLICAS", "2")
subprocess.check_call(
[
"docker",
"compose",
"--env-file",
".env",
"up",
"-d",
"--no-deps",
"--scale",
f"api={replicas}",
"--remove-orphans",
"api",
],
cwd=ROOT,
)
# Public ready is 200 via the surviving replica; wait until the LB routes to the restored one.
for _ in range(180):
try:
if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError(f"replica {target} not back in LB rotation after restore")
def one_round(round_no: int) -> dict:
uname = "seq_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call(
"POST",
"/api/v1/users",
{"username": uname, "display_name": "Seq", "password": pw},
expected=201,
)["json"]
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + cred},
expected=201,
)["json"]["token"]
peer_u = "seqb_" + uuid.uuid4().hex[:10]
peer = call(
"POST",
"/api/v1/users",
{"username": peer_u, "display_name": "SeqB", "password": pw},
expected=201,
)["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "seq", "member_ids": [user["id"], peer["id"]]},
token=tok,
expected=201,
)["json"]
observed = set()
for _ in range(40):
observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id"))
if len(observed) < 2:
raise RuntimeError(f"need ≥2 instances, got {observed}")
target = sorted(observed)[round_no % len(observed)]
env = dict(os.environ)
env["TARGET_INSTANCE_ID"] = target
script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh")
started = time.monotonic()
proc = subprocess.Popen([str(script)], env=env, cwd=ROOT)
ok = fail = 0
first = None
after = []
while time.monotonic() - started < 20:
code = proc.poll()
try:
key = uuid.uuid4().hex
r = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": "seq-" + key},
token=tok,
headers={"Idempotency-Key": key},
expected=201,
)
ok += 1
if code == 0:
if first is None:
first = time.monotonic() - started
after.append(header(r, "X-Instance-Id"))
except Exception:
fail += 1
time.sleep(0.1)
if proc.poll() is None:
proc.wait(timeout=10)
result = {
"round": round_no,
"observed_before": sorted(observed),
"target": target,
"successful_writes": ok,
"failed_attempts": fail,
"first_recovery_seconds": round(first, 3) if first is not None else None,
"target_absent_in_last5": target not in after[-5:],
}
t0 = time.monotonic()
restore_api(target)
result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3)
return result
def main():
results = []
for i in range(2):
results.append(one_round(i))
print(json.dumps(results[-1], ensure_ascii=False))
out = ROOT / "evidence" / "seq-failover.json"
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print("WROTE", out)
if __name__ == "__main__":
main()