Files
mtusi2026labs/scripts/probe.py
T
ArtyomandClaude Opus 5.5 e927afdb09 Carry over the lab 3 fixes made after 61c0c28
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4
code, resolving each conflict by meaning and keeping everything lab 4
added:

- a message is posted with one SQL statement under the chat row lock,
  and the attachment links go into the same statement; a foreign key
  violation on the attachment link still answers 404 and leaves nothing
  behind. Text and the attachment list are checked before the write, an
  outsider gets 404 for both, as before the carry over;
- the request pipeline counts the body bytes actually received against
  the limit of the route, so a chunked body over 1 MiB gets 413 and the
  upload route keeps its MAX_UPLOAD_BYTES limit;
- a request cancelled at shutdown gets 503 unavailable in the contract
  format, the worker waits a second less than the rest of the shutdown
  budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s;
- readiness sits behind a Readiness port with the draining flag in
  shared infrastructure, and the drain window is logged through Logger;
- logout revokes through SessionRevocations and withdraws the mark when
  PostgreSQL rejects the delete; an empty cursor is 400;
- EventBus.project is gone, the sweep test subscribes and waits on a
  queue; domain tests check attachment_list instead of Chat.post;
- make up waits for every replica in the balancer, BASE_URL follows the
  lb port, the image is tagged by the compose project, retry-check and
  faults-evidence are added, detection runs three times, failover-each
  and rolling-update record the stand usage, CI checks numeric uids.
  down also takes the tools and reconcile profiles of lab 4.

Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU
pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the
database size itself instead of the track 1 sampler.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-30 06:25:23 +03:00

541 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import base64
import collections
import functools
import json
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from datetime import datetime
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
PSQL += ["-X", "-q", "-At"]
@functools.cache
def published(port, fallback):
command = ["docker", "compose", "port", "lb", str(port)]
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return found.stdout.strip() or fallback
def base_url():
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
def stats_url():
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
def call(method, path, body=None, headers=None, timeout=2.0):
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
head = {"Accept": "application/json"}
if data is not None:
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(
base_url().rstrip("/") + path, data=data, headers=head, method=method
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read() or b"null")
return response.status, response.headers.get("X-Instance-Id", ""), payload
except urllib.error.HTTPError as error:
try:
payload = json.loads(error.read() or b"null")
except (OSError, ValueError):
payload = None
return error.code, error.headers.get("X-Instance-Id", ""), payload
except (OSError, ValueError):
return 0, "", None
def emit(report, out=None):
text = json.dumps(report, ensure_ascii=False, indent=2)
if out:
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out).write_text(text + "\n")
print(text)
def created(result, what):
if result[0] != 201:
raise SystemExit(f"{what} failed: HTTP {result[0]}")
return result[2]
def actor(label):
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
password = "Probe-" + uuid.uuid4().hex
body = {"username": username, "display_name": "probe " + label, "password": password}
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
login = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
)
return {
"id": user["id"],
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
}
def chat_of(owner, other):
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
def send(owner, chat, text, key):
headers = dict(owner["auth"], **{"Idempotency-Key": key})
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
def lb_rows(backend="api"):
try:
with urllib.request.urlopen(stats_url(), timeout=2) as response:
lines = response.read().decode().splitlines()
except OSError:
return None
header = lines[0].lstrip("# ").split(",")
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
return [r for r in rows if r["pxname"] == backend]
def lb_up():
rows = lb_rows()
if rows is None:
return None
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
def lb_counters():
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
return {name: int(row.get(name) or 0) for name in COUNTERS}
def run_k6(out, env, on_start=None):
Path(out).mkdir(parents=True, exist_ok=True)
started = []
process = subprocess.Popen(
[str(ROOT / "scripts/k6.sh"), str(out)],
env=dict(os.environ, **env),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
with (Path(out) / "k6.txt").open("w") as log:
for line in process.stdout:
log.write(line)
if not started and "LOAD-START" in line:
started.append(time.time())
if on_start:
on_start()
return process.wait(), started[0] if started else None
def k6_metrics(path, duration):
metrics = json.loads(Path(path).read_text())["metrics"]
def value(name, field):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
count = value("http_reqs", "count")
return {
"requests": int(count),
"rps": round(count / duration, 2),
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
"failed_ratio": round(value("http_req_failed", "rate"), 5),
"dropped_iterations": int(value("dropped_iterations", "count")),
"p50_ms": round(value("http_req_duration", "med"), 2),
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
}
def race(alice, chat, parallel):
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
bodies = [r[2] for r in results if r[0] == 201]
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
return text, results, identical, send(alice, chat, text + " changed", key)[0]
def check(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
seen = collections.Counter(
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
)
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
same = {
"rounds": len(rounds),
"parallel": args.parallel,
"statuses": dict(statuses),
"instances": dict(served),
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
}
passed = (
len(seen) >= 2
and statuses == {201: len(rounds) * args.parallel}
and len(served) >= 2
and same["identical_bodies_rounds"] == len(rounds)
and same["single_copy_rounds"] == len(rounds)
and same["conflict_rounds"] == len(rounds)
)
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
return 0 if passed else 1
def output(*command):
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return done.stdout + done.stderr
def replicas():
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
def answered(container, request_id):
for line in output("docker", "logs", container).splitlines():
if request_id in line and "request.handled" in line:
return json.loads(line).get("status")
return None
def lost(alice, bob, chat, kill, args):
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
request_id = "lost-" + key[:12]
lock = subprocess.Popen(
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
)
lock.stdin.flush()
lock.stdout.readline()
sent = time.time()
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
path = f"/api/v1/chats/{chat}/messages"
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
waiting = set(output(*PSQL, "-c", sql).split())
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
if kill and holder[0]:
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
lock.stdin.close()
lock.wait()
retried = time.time()
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
while status != 201 and attempts < 20:
time.sleep(0.2)
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
copies = [m for m in page["items"] if m["text"] == text]
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
report = {
"variant": "kill" if kill else "timeout",
"lost_attempt": {
"status": first,
"client_timeout_s": args.timeout,
"chat_lock_held_s": args.hold,
"replica": holder[1],
"replica_killed": bool(kill and holder[0]),
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
},
"retry": {
"after_s": round(retried - sent, 3),
"status": status,
"attempts": attempts,
"replica": instance,
},
"copies_in_history": len(copies),
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
"written_by": None
if stamp is None
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
}
if kill and holder[0]:
subprocess.call(
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
report["pass"] = (
first == 0
and holder[0] is not None
and status == 201
and report["copies_in_history"] == 1
and report["same_message"]
and (kill or report["lost_attempt"]["replica_answered"] == 201)
)
return report
def retry(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
passed = all(v["pass"] for v in variants)
emit({"variants": variants, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
key = uuid.uuid4().hex
time.sleep(0.1)
def poller():
while not stop.is_set():
trace.append((time.time(), lb_up()))
time.sleep(0.1)
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
def write_csv(path, header, rows):
with path.open("w") as handle:
handle.write(header + "\n")
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
served = [(t, i) for t, s, _, i in probes if s == 201]
before = collections.Counter(i for t, i in served if t < t0)
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
after = collections.Counter(i for t, i in served if t > end)
return {
"label": label,
"action": args.command,
"action_exit": action["exit"],
"action_seconds": round(end - t0, 3),
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
},
}
def verdict(args, result):
probe, k6 = result["probe"], result["k6"]
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
probe["instances_after"].get(result["label"])
)
if args.expect == "replace":
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
old, new = set(probe["instances_before"]), set(probe["instances_after"])
checks["all_replaced"] = bool(new) and not old & new
return checks
def load(args):
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
def act():
time.sleep(args.warmup + args.lead)
action["start"] = time.time()
with (out / "action.txt").open("w") as log:
action["exit"] = subprocess.call(
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
)
action["end"] = time.time()
runner = threading.Thread(target=act)
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
counters = lb_counters()
code, load_start = run_k6(out / "k6", env, runner.start)
if load_start is not None:
runner.join()
action["load_end"] = load_start + args.warmup + args.duration
stop.set()
for thread in threads:
thread.join()
if "start" not in action:
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
t0 = action["start"]
write_csv(
out / "probe.csv",
"t_rel_s,status,latency_ms,instance",
((round(t - t0, 3), *r) for t, *r in probes),
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
)
result["checks"] = verdict(args, result)
result["pass"] = all(result["checks"].values())
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False))
return 0 if result["pass"] else 1
def detect(args):
before = lb_up() or 0
until = before - 1 if args.until is None else args.until
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
acted = time.time() - started if acted is None else acted
status, instance, payload = call("GET", "/health/live")
restored = None
if args.restore:
begun = time.time()
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
time.sleep(0.1)
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
if args.within_action:
checks["excluded_before_action_end"] = detected is not None and detected < acted
if args.restore:
checks["restored"] = restored is not None
report = {
"action": args.command,
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
"measurement_slack_seconds": args.slack,
"lb_answer_after_detection": {
"path": "/health/live",
"status": status,
"x_instance_id_present": bool(instance),
"error_code": ((payload or {}).get("error") or {}).get("code"),
},
"restore": args.restore,
"restore_seconds": restored,
"lb_up_after_restore": lb_up(),
"checks": checks,
"pass": all(checks.values()),
}
emit(report, args.out)
return 0 if report["pass"] else 1
def main():
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
sub = parser.add_subparsers(dest="mode", required=True)
one = sub.add_parser("check", help="instances behind one token and same-key races")
one.add_argument("--samples", type=int, default=40)
one.add_argument("--parallel", type=int, default=8)
one.add_argument("--rounds", type=int, default=1)
one.add_argument("--out", help="JSON file, written only after a complete report")
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
under.add_argument("--out", required=True)
under.add_argument("--label")
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
under.add_argument("--warmup", type=int, default=10)
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
under.add_argument("--lead", type=int, default=15)
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
again.add_argument("--out", help="JSON file, written only after a complete report")
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
watch.add_argument("--until", type=int, help="UP servers counted as detected")
watch.add_argument("--within-action", action="store_true")
watch.add_argument("--lb-status", type=int, default=200)
watch.add_argument("--restore", help="shell command that brings replicas back")
watch.add_argument("--timeout", type=float, default=20)
watch.add_argument("--out", help="JSON file, written only after a complete report")
watch.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.mode == "check":
return check(args)
if args.mode == "retry":
return retry(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
return load(args) if args.mode == "load" else detect(args)
if __name__ == "__main__":
sys.exit(main())