forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
171 lines
5.8 KiB
Python
171 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-4: closed-loop POSTs during rolling API replacement."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
REPLICAS = os.environ.get("API_REPLICAS", "2")
|
|
|
|
|
|
def call(method, path, body=None, token=None, headers=None):
|
|
h = {"Accept": "application/json"}
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
h["Content-Type"] = "application/json"
|
|
if headers:
|
|
h.update(headers)
|
|
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
|
try:
|
|
resp = urllib.request.urlopen(req, timeout=5)
|
|
except urllib.error.HTTPError as err:
|
|
resp = err
|
|
except Exception as exc:
|
|
return {"status": 0, "error": type(exc).__name__}
|
|
raw = resp.read()
|
|
js = None
|
|
if raw and resp.headers.get_content_type() == "application/json":
|
|
js = json.loads(raw.decode("utf-8"))
|
|
return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")}
|
|
|
|
|
|
def api_ids():
|
|
out = subprocess.check_output(
|
|
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True
|
|
)
|
|
return out.split()
|
|
|
|
|
|
def hostname(cid):
|
|
return subprocess.check_output(
|
|
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
|
|
).strip()
|
|
|
|
|
|
def replace_replica(old_id, known):
|
|
"""Stop+remove one replica, create a fresh one, wait until the LB routes to it."""
|
|
subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL)
|
|
subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL)
|
|
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
|
|
subprocess.check_call(
|
|
["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate",
|
|
"--scale", f"api={REPLICAS}", "api"],
|
|
cwd=ROOT,
|
|
)
|
|
new = [cid for cid in api_ids() if cid not in known]
|
|
if len(new) != 1:
|
|
raise RuntimeError(f"expected one new replica, got {new}")
|
|
host = hostname(new[0])
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
r = call("GET", "/health/ready")
|
|
if r.get("status") == 200 and r.get("instance") == host:
|
|
return new[0], host
|
|
time.sleep(0.5)
|
|
raise RuntimeError(f"new replica {host} not in LB rotation after 90s")
|
|
|
|
|
|
def bootstrap():
|
|
uname = "roll_" + uuid.uuid4().hex[:10]
|
|
pw = "T3st-" + uuid.uuid4().hex
|
|
user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw})
|
|
assert user["status"] == 201, user
|
|
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
|
|
tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"]
|
|
peer_u = "rollb_" + uuid.uuid4().hex[:10]
|
|
peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"]
|
|
chat = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
{"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]},
|
|
token=tok,
|
|
)["json"]
|
|
return tok, chat["id"]
|
|
|
|
|
|
def main():
|
|
tok, chat_id = bootstrap()
|
|
stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []}
|
|
stop = threading.Event()
|
|
|
|
def worker():
|
|
while not stop.is_set():
|
|
key = uuid.uuid4().hex
|
|
t0 = time.monotonic()
|
|
r = call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat_id}/messages",
|
|
{"text": "roll-" + key},
|
|
token=tok,
|
|
headers={"Idempotency-Key": key},
|
|
)
|
|
dt = (time.monotonic() - t0) * 1000
|
|
if r.get("status") == 201:
|
|
stats["ok"] += 1
|
|
stats["latencies_ms"].append(dt)
|
|
else:
|
|
stats["err"] += 1
|
|
msg = ((r.get("json") or {}).get("error") or {}).get("message", "")
|
|
code = f"{r.get('status') or r.get('error')} {msg}".strip()
|
|
stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1
|
|
time.sleep(0.05)
|
|
|
|
ids = api_ids()
|
|
known = set(ids)
|
|
replaced = []
|
|
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
|
|
started = time.monotonic()
|
|
for th in threads:
|
|
th.start()
|
|
time.sleep(2) # warm-up
|
|
# Rolling: replace each original replica in turn; the next one is stopped
|
|
# only after the LB already routes requests to the fresh container.
|
|
for cid in ids:
|
|
old_host = hostname(cid)
|
|
new_id, new_host = replace_replica(cid, known)
|
|
known.add(new_id)
|
|
replaced.append({"old": old_host, "new": new_host})
|
|
time.sleep(2)
|
|
time.sleep(2)
|
|
stop.set()
|
|
for th in threads:
|
|
th.join(timeout=5)
|
|
duration = time.monotonic() - started
|
|
lats = sorted(stats["latencies_ms"])
|
|
def pct(p):
|
|
if not lats:
|
|
return None
|
|
return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2)
|
|
summary = {
|
|
"successful": stats["ok"],
|
|
"errors": stats["err"],
|
|
"errors_by_status": stats["err_by_status"],
|
|
"p50_ms": pct(0.50),
|
|
"p95_ms": pct(0.95),
|
|
"p99_ms": pct(0.99),
|
|
"duration_s": round(duration, 1),
|
|
"replicas_rolled": len(replaced),
|
|
"replaced": replaced,
|
|
}
|
|
out = ROOT / "evidence" / "rolling-load.json"
|
|
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps(summary, ensure_ascii=False))
|
|
print("WROTE", out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|