Files
mtusi2026lab3/scripts/check_503_retry_after.py
T

161 lines
5.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import argparse
import base64
import json
import statistics
import sys
import time
import urllib.error
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
BASE = "http://localhost:8080"
def setup():
pwd = "SecurePassword123!"
ids = []
for _ in range(2):
u = f"ra_{uuid.uuid4().hex[:10]}"
s, _, user, _ = call("POST", "/api/v1/users",
body={"username": u, "display_name": "RA", "password": pwd})
assert s == 201, (s, user)
ids.append((u, user["id"]))
cred = base64.b64encode(f"{ids[0][0]}:{pwd}".encode()).decode()
s, _, sess, _ = call("POST", "/api/v1/auth/sessions",
headers={"Authorization": f"Basic {cred}"})
assert s == 201, (s, sess)
h = {"Authorization": f"Bearer {sess['token']}"}
s, _, chat, _ = call("POST", "/api/v1/chats", headers=h,
body={"title": "Retry-After check", "member_ids": [i[1] for i in ids]})
assert s == 201, (s, chat)
return h, chat["id"]
def pct(vals, p):
if not vals:
return None
v = sorted(vals)
return round(1000 * v[min(len(v) - 1, int(p * len(v)))], 1)
def call(method, path, headers=None, body=None):
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(BASE + path, data=data, method=method)
req.add_header("Accept", "application/json")
if data:
req.add_header("Content-Type", "application/json")
for k, v in (headers or {}).items():
req.add_header(k, v)
t0 = time.monotonic()
try:
with urllib.request.urlopen(req, timeout=60) as r:
raw = r.read().decode()
hdr = {k.lower(): v for k, v in r.headers.items()}
return r.status, hdr, json.loads(raw) if raw else None, time.monotonic() - t0
except urllib.error.HTTPError as e:
raw = e.read().decode()
hdr = {k.lower(): v for k, v in e.headers.items()}
try:
parsed = json.loads(raw) if raw else None
except ValueError:
parsed = {"_raw": raw[:200]}
return e.code, hdr, parsed, time.monotonic() - t0
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--n", type=int, default=64)
ap.add_argument("--allow-zero-503", action="store_true")
args = ap.parse_args()
n = args.n
h, chat_id = setup()
keys = [f"ra-{uuid.uuid4().hex}" for _ in range(n)]
bodies = [{"text": f"retry-after probe {i}"} for i in range(n)]
def send(i):
s, hdr, body, dt = call("POST", f"/api/v1/chats/{chat_id}/messages",
headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i])
return i, s, hdr, body, dt
wall0 = time.monotonic()
with ThreadPoolExecutor(max_workers=n) as ex:
first = list(ex.map(send, range(n)))
wall = time.monotonic() - wall0
codes, problems, retry_idx, lat = {}, [], [], []
instances = set()
for i, s, hdr, body, dt in first:
codes[s] = codes.get(s, 0) + 1
lat.append(dt)
if hdr.get("x-instance-id"):
instances.add(hdr["x-instance-id"])
if s == 503:
retry_idx.append(i)
ra = hdr.get("retry-after")
err = (body or {}).get("error", {})
if err.get("code") != "unavailable":
problems.append(f"[{i}] code={err.get('code')}, ожидался unavailable")
if not ra or not ra.isdigit() or int(ra) < 1:
problems.append(f"[{i}] некорректный Retry-After: {ra!r}")
if err.get("request_id") != hdr.get("x-request-id"):
problems.append(f"[{i}] request_id в теле != заголовку")
if not hdr.get("x-instance-id"):
problems.append(f"[{i}] нет X-Instance-Id")
if "QueuePool" in json.dumps(body):
problems.append(f"[{i}] утечка внутренних деталей")
if s == 500:
problems.append(f"[{i}] 500 instead of 503: {body}")
print(f"Concurrent: {n}; wall time: {wall:.2f}s; replices found: {len(instances)} {sorted(instances)}")
print(f"Statusues of 1 cycle: {codes}")
print(f"Latency, ms: p50={pct(lat,.5)} p95={pct(lat,.95)} p99={pct(lat,.99)}")
print(f"503 с Retry-After: {len(retry_idx)}")
if retry_idx:
smp = next(r for r in first if r[1] == 503)
print(f"Example: Retry-After={smp[2].get('retry-after')}, "
f"message={smp[3]['error']['message']!r}")
ok_ids = {b["id"] for _, s, _, b, _ in first if s == 201}
for i in retry_idx:
s, hdr, body, _ = call("POST", f"/api/v1/chats/{chat_id}/messages",
headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i])
if s != 201:
problems.append(f"retry[{i}] -> {s}, expected 201: {body}")
else:
ok_ids.add(body["id"])
total, cursor = 0, None
while True:
p = f"/api/v1/chats/{chat_id}/messages?limit=100"
if cursor:
p += "&cursor=" + cursor
s, _, page, _ = call("GET", p, headers=h)
assert s == 200, (s, page)
total += len(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
print(f"Messages in chat: {total}; unique id: {len(ok_ids)}; keys: {n}")
if total != n or len(ok_ids) != n:
problems.append(f"dublicates/losses: total={total}, ids={len(ok_ids)}, expected {n}")
if problems:
print("\nFAIL:")
for p in problems[:15]:
print(" -", p)
if len(problems) > 15:
print(f" ... и ещё {len(problems)-15}")
sys.exit(1)
if not retry_idx and not args.allow_zero_503:
print("\nINCONCLUSIVE: 503 не наблюдалось — маппинг пула НЕ проверен.")
sys.exit(2)
print("\nPASS: 503 unavailable + Retry-After correct, retries are safe, no dublicates")
if __name__ == "__main__":
main()