scripts/stop-one.sh maps TARGET_INSTANCE_ID to the api container whose hostname matches, stops only it with docker stop -t 6 and leaves it stopped; an unknown id exits 3 and touches nothing. scripts/restart.sh replaces the lab 2 hook: it stops lb and the replicas, restarts PostgreSQL and Valkey with their volumes and brings the stand back with up --wait, which also reruns the idempotent migrate job. scripts/restore.sh starts stopped replicas without recreating them and scripts/lb-wait.sh waits until the balancer statistics show every replica UP again, so the next failover run sees two replicas. scripts/probe.py check (stdlib, Python 3.9 syntax) counts X-Instance-Id over 40 calls with one token and races 8 posts with one key through the balancer. make restore and make lb-check run them, ruff covers the probe, and the CI acceptance job runs the failover test and the probe. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
139 lines
4.8 KiB
Python
Executable File
139 lines
4.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import functools
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
@functools.cache
|
|
def published(port, fallback):
|
|
command = ["docker", "compose", "port", "lb", str(port)]
|
|
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return found.stdout.strip() or fallback
|
|
|
|
|
|
def base_url():
|
|
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
|
|
|
|
|
def call(method, path, body=None, headers=None, timeout=2.0):
|
|
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
|
head = {"Accept": "application/json"}
|
|
if data is not None:
|
|
head["Content-Type"] = "application/json"
|
|
head.update(headers or {})
|
|
request = urllib.request.Request(
|
|
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.loads(response.read() or b"null")
|
|
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.headers.get("X-Instance-Id", ""), None
|
|
except (OSError, ValueError):
|
|
return 0, "", None
|
|
|
|
|
|
def emit(report, out=None):
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if out:
|
|
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(out).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
def created(result, what):
|
|
if result[0] != 201:
|
|
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
|
return result[2]
|
|
|
|
|
|
def actor(label):
|
|
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Probe-" + uuid.uuid4().hex
|
|
body = {"username": username, "display_name": "probe " + label, "password": password}
|
|
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
login = call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
|
)
|
|
return {
|
|
"id": user["id"],
|
|
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
|
}
|
|
|
|
|
|
def chat_of(owner, other):
|
|
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
|
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
|
|
|
|
|
def send(owner, chat, text, key):
|
|
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
|
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
|
|
|
|
|
def check(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
seen = collections.Counter(
|
|
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
|
)
|
|
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
|
barrier, results = threading.Barrier(args.parallel), []
|
|
|
|
def worker():
|
|
barrier.wait()
|
|
results.append(send(alice, chat, text, key))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(args.parallel)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
|
bodies = [r[2] for r in results if r[0] == 201]
|
|
race = {
|
|
"parallel": args.parallel,
|
|
"statuses": dict(collections.Counter(r[0] for r in results)),
|
|
"instances": dict(collections.Counter(r[1] for r in results)),
|
|
"identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies),
|
|
"stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text),
|
|
"other_body_status": send(alice, chat, text + " changed", key)[0],
|
|
}
|
|
passed = (
|
|
len(seen) >= 2
|
|
and race["statuses"] == {201: args.parallel}
|
|
and race["identical_bodies"]
|
|
and race["stored_copies"] == 1
|
|
and race["other_body_status"] == 409
|
|
)
|
|
emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
|
sub = parser.add_subparsers(dest="mode", required=True)
|
|
one = sub.add_parser("check", help="instances behind one token and a same-key race")
|
|
one.add_argument("--samples", type=int, default=40)
|
|
one.add_argument("--parallel", type=int, default=8)
|
|
one.add_argument("--out", help="JSON file, written only after a complete report")
|
|
return check(parser.parse_args())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|