forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
172 lines
5.2 KiB
Python
172 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-4 evidence: stop each observed API replica in turn; restore between steps."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
|
|
|
|
def call(method, path, body=None, token=None, headers=None, expected=None):
|
|
h = {"Accept": "application/json"}
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
h["Content-Type"] = "application/json"
|
|
if headers:
|
|
h.update(headers)
|
|
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
|
try:
|
|
resp = urllib.request.urlopen(req, timeout=5)
|
|
except urllib.error.HTTPError as err:
|
|
resp = err
|
|
raw = resp.read()
|
|
status = resp.code
|
|
if expected is not None and status != expected:
|
|
raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}")
|
|
js = None
|
|
if raw and resp.headers.get_content_type() == "application/json":
|
|
js = json.loads(raw.decode("utf-8"))
|
|
return {"status": status, "headers": resp.headers, "json": js}
|
|
|
|
|
|
def header(resp, name: str) -> str:
|
|
return resp["headers"].get(name) or resp["headers"].get(name.lower()) or ""
|
|
|
|
|
|
def restore_api(target: str):
|
|
replicas = os.environ.get("API_REPLICAS", "2")
|
|
subprocess.check_call(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
".env",
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--scale",
|
|
f"api={replicas}",
|
|
"--remove-orphans",
|
|
"api",
|
|
],
|
|
cwd=ROOT,
|
|
)
|
|
# Public ready is 200 via the surviving replica; wait until the LB routes to the restored one.
|
|
for _ in range(180):
|
|
try:
|
|
if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target:
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise RuntimeError(f"replica {target} not back in LB rotation after restore")
|
|
|
|
|
|
def one_round(round_no: int) -> dict:
|
|
uname = "seq_" + uuid.uuid4().hex[:10]
|
|
pw = "T3st-" + uuid.uuid4().hex
|
|
user = call(
|
|
"POST",
|
|
"/api/v1/users",
|
|
{"username": uname, "display_name": "Seq", "password": pw},
|
|
expected=201,
|
|
)["json"]
|
|
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
|
|
tok = call(
|
|
"POST",
|
|
"/api/v1/auth/sessions",
|
|
headers={"Authorization": "Basic " + cred},
|
|
expected=201,
|
|
)["json"]["token"]
|
|
peer_u = "seqb_" + uuid.uuid4().hex[:10]
|
|
peer = call(
|
|
"POST",
|
|
"/api/v1/users",
|
|
{"username": peer_u, "display_name": "SeqB", "password": pw},
|
|
expected=201,
|
|
)["json"]
|
|
chat = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
{"title": "seq", "member_ids": [user["id"], peer["id"]]},
|
|
token=tok,
|
|
expected=201,
|
|
)["json"]
|
|
|
|
observed = set()
|
|
for _ in range(40):
|
|
observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id"))
|
|
if len(observed) < 2:
|
|
raise RuntimeError(f"need ≥2 instances, got {observed}")
|
|
target = sorted(observed)[round_no % len(observed)]
|
|
|
|
env = dict(os.environ)
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh")
|
|
started = time.monotonic()
|
|
proc = subprocess.Popen([str(script)], env=env, cwd=ROOT)
|
|
ok = fail = 0
|
|
first = None
|
|
after = []
|
|
while time.monotonic() - started < 20:
|
|
code = proc.poll()
|
|
try:
|
|
key = uuid.uuid4().hex
|
|
r = call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
{"text": "seq-" + key},
|
|
token=tok,
|
|
headers={"Idempotency-Key": key},
|
|
expected=201,
|
|
)
|
|
ok += 1
|
|
if code == 0:
|
|
if first is None:
|
|
first = time.monotonic() - started
|
|
after.append(header(r, "X-Instance-Id"))
|
|
except Exception:
|
|
fail += 1
|
|
time.sleep(0.1)
|
|
if proc.poll() is None:
|
|
proc.wait(timeout=10)
|
|
result = {
|
|
"round": round_no,
|
|
"observed_before": sorted(observed),
|
|
"target": target,
|
|
"successful_writes": ok,
|
|
"failed_attempts": fail,
|
|
"first_recovery_seconds": round(first, 3) if first is not None else None,
|
|
"target_absent_in_last5": target not in after[-5:],
|
|
}
|
|
t0 = time.monotonic()
|
|
restore_api(target)
|
|
result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3)
|
|
return result
|
|
|
|
|
|
def main():
|
|
results = []
|
|
for i in range(2):
|
|
results.append(one_round(i))
|
|
print(json.dumps(results[-1], ensure_ascii=False))
|
|
out = ROOT / "evidence" / "seq-failover.json"
|
|
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print("WROTE", out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|