Files
mtusi2026labs/scripts/probe.py
T
ArtyomandClaude Opus 5.5 ceb266ed05 Add the stop-one, restart and restore hooks and an LB probe
scripts/stop-one.sh maps TARGET_INSTANCE_ID to the api container whose
hostname matches, stops only it with docker stop -t 6 and leaves it
stopped; an unknown id exits 3 and touches nothing. scripts/restart.sh
replaces the lab 2 hook: it stops lb and the replicas, restarts
PostgreSQL and Valkey with their volumes and brings the stand back
with up --wait, which also reruns the idempotent migrate job.
scripts/restore.sh starts stopped replicas without recreating them
and scripts/lb-wait.sh waits until the balancer statistics show every
replica UP again, so the next failover run sees two replicas.

scripts/probe.py check (stdlib, Python 3.9 syntax) counts X-Instance-Id
over 40 calls with one token and races 8 posts with one key through
the balancer. make restore and make lb-check run them, ruff covers the
probe, and the CI acceptance job runs the failover test and the probe.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-28 11:43:52 +03:00

139 lines
4.8 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import base64
import collections
import functools
import json
import os
import subprocess
import sys
import threading
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
@functools.cache
def published(port, fallback):
command = ["docker", "compose", "port", "lb", str(port)]
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
return found.stdout.strip() or fallback
def base_url():
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
def call(method, path, body=None, headers=None, timeout=2.0):
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
head = {"Accept": "application/json"}
if data is not None:
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(
base_url().rstrip("/") + path, data=data, headers=head, method=method
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read() or b"null")
return response.status, response.headers.get("X-Instance-Id", ""), payload
except urllib.error.HTTPError as error:
return error.code, error.headers.get("X-Instance-Id", ""), None
except (OSError, ValueError):
return 0, "", None
def emit(report, out=None):
text = json.dumps(report, ensure_ascii=False, indent=2)
if out:
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out).write_text(text + "\n")
print(text)
def created(result, what):
if result[0] != 201:
raise SystemExit(f"{what} failed: HTTP {result[0]}")
return result[2]
def actor(label):
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
password = "Probe-" + uuid.uuid4().hex
body = {"username": username, "display_name": "probe " + label, "password": password}
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
login = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
)
return {
"id": user["id"],
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
}
def chat_of(owner, other):
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
def send(owner, chat, text, key):
headers = dict(owner["auth"], **{"Idempotency-Key": key})
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
def check(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
seen = collections.Counter(
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
)
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(args.parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(args.parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
bodies = [r[2] for r in results if r[0] == 201]
race = {
"parallel": args.parallel,
"statuses": dict(collections.Counter(r[0] for r in results)),
"instances": dict(collections.Counter(r[1] for r in results)),
"identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies),
"stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text),
"other_body_status": send(alice, chat, text + " changed", key)[0],
}
passed = (
len(seen) >= 2
and race["statuses"] == {201: args.parallel}
and race["identical_bodies"]
and race["stored_copies"] == 1
and race["other_body_status"] == 409
)
emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out)
return 0 if passed else 1
def main():
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
sub = parser.add_subparsers(dest="mode", required=True)
one = sub.add_parser("check", help="instances behind one token and a same-key race")
one.add_argument("--samples", type=int, default=40)
one.add_argument("--parallel", type=int, default=8)
one.add_argument("--out", help="JSON file, written only after a complete report")
return check(parser.parse_args())
if __name__ == "__main__":
sys.exit(main())