Files
mtusi2026lab4/scripts/rolling_load_evidence.py
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

171 lines
5.8 KiB
Python

#!/usr/bin/env python3
"""Grade-4: closed-loop POSTs during rolling API replacement."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import threading
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
REPLICAS = os.environ.get("API_REPLICAS", "2")
def call(method, path, body=None, token=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
except Exception as exc:
return {"status": 0, "error": type(exc).__name__}
raw = resp.read()
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")}
def api_ids():
out = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True
)
return out.split()
def hostname(cid):
return subprocess.check_output(
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
).strip()
def replace_replica(old_id, known):
"""Stop+remove one replica, create a fresh one, wait until the LB routes to it."""
subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL)
subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL)
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
subprocess.check_call(
["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate",
"--scale", f"api={REPLICAS}", "api"],
cwd=ROOT,
)
new = [cid for cid in api_ids() if cid not in known]
if len(new) != 1:
raise RuntimeError(f"expected one new replica, got {new}")
host = hostname(new[0])
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
r = call("GET", "/health/ready")
if r.get("status") == 200 and r.get("instance") == host:
return new[0], host
time.sleep(0.5)
raise RuntimeError(f"new replica {host} not in LB rotation after 90s")
def bootstrap():
uname = "roll_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw})
assert user["status"] == 201, user
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"]
peer_u = "rollb_" + uuid.uuid4().hex[:10]
peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]},
token=tok,
)["json"]
return tok, chat["id"]
def main():
tok, chat_id = bootstrap()
stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []}
stop = threading.Event()
def worker():
while not stop.is_set():
key = uuid.uuid4().hex
t0 = time.monotonic()
r = call(
"POST",
f"/api/v1/chats/{chat_id}/messages",
{"text": "roll-" + key},
token=tok,
headers={"Idempotency-Key": key},
)
dt = (time.monotonic() - t0) * 1000
if r.get("status") == 201:
stats["ok"] += 1
stats["latencies_ms"].append(dt)
else:
stats["err"] += 1
msg = ((r.get("json") or {}).get("error") or {}).get("message", "")
code = f"{r.get('status') or r.get('error')} {msg}".strip()
stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1
time.sleep(0.05)
ids = api_ids()
known = set(ids)
replaced = []
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
started = time.monotonic()
for th in threads:
th.start()
time.sleep(2) # warm-up
# Rolling: replace each original replica in turn; the next one is stopped
# only after the LB already routes requests to the fresh container.
for cid in ids:
old_host = hostname(cid)
new_id, new_host = replace_replica(cid, known)
known.add(new_id)
replaced.append({"old": old_host, "new": new_host})
time.sleep(2)
time.sleep(2)
stop.set()
for th in threads:
th.join(timeout=5)
duration = time.monotonic() - started
lats = sorted(stats["latencies_ms"])
def pct(p):
if not lats:
return None
return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2)
summary = {
"successful": stats["ok"],
"errors": stats["err"],
"errors_by_status": stats["err_by_status"],
"p50_ms": pct(0.50),
"p95_ms": pct(0.95),
"p99_ms": pct(0.99),
"duration_s": round(duration, 1),
"replicas_rolled": len(replaced),
"replaced": replaced,
}
out = ROOT / "evidence" / "rolling-load.json"
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(summary, ensure_ascii=False))
print("WROTE", out)
if __name__ == "__main__":
main()