Files
mtusi2026lab3/scripts/measure_failover.py
T
2026-10-22 18:41:56 +03:00

91 lines
4.1 KiB
Python

import base64, json, subprocess, sys, threading, time, urllib.request, uuid
import os
BASE = os.environ.get("BASE", "http://localhost:8080"); PWD = "correct-horse-battery-staple"
WORKERS, DURATION, STOP_AT, START_AT = 4, 35, 8, 22
def req(method, path, body=None, token=None, basic=None, key=None):
r = urllib.request.Request(BASE + path, method=method,
data=json.dumps(body).encode() if body is not None else None)
if body is not None: r.add_header("Content-Type", "application/json")
if token: r.add_header("Authorization", "Bearer " + token)
if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode())
if key: r.add_header("Idempotency-Key", key)
with urllib.request.urlopen(r, timeout=3) as resp:
return resp.status, resp.headers.get("X-Instance-Id"), json.loads(resp.read() or b"null")
def sh(*a): return subprocess.run(a, capture_output=True, text=True).stdout.split()
arg = sys.argv[1]
names = sorted(sh("docker", "ps", "--filter", "label=com.docker.compose.service=api",
"--format", "{{.Names}}"))
container = names[int(arg) - 1] if arg.isdigit() else arg
stopped_id = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", container],
capture_output=True, text=True).stdout.strip()
assert stopped_id, "container not found"
name = "fo_" + uuid.uuid4().hex[:10]
_, _, u = req("POST", "/api/v1/users", {"username": name, "display_name": "fo", "password": PWD})
_, _, s = req("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}")
tok = s["token"]
_, _, c = req("POST", "/api/v1/chats", {"title": "fo", "member_ids": [u["id"]]}, token=tok)
path = f"/api/v1/chats/{c['id']}/messages"
t0 = time.time()
attempts, confirmed, unconfirmed = [], set(), 0
lock = threading.Lock()
def action():
time.sleep(STOP_AT)
subprocess.run(["docker", "stop", "-t", "8", container], capture_output=True)
time.sleep(max(0, START_AT - (time.time() - t0)))
subprocess.run(["docker", "start", container], capture_output=True)
def worker():
global unconfirmed
while time.time() - t0 < DURATION:
key = uuid.uuid4().hex
done = False
for n in range(3):
ts = time.time() - t0
try:
_, inst, m = req("POST", path, {"text": "x"}, token=tok, key=key)
with lock:
attempts.append((ts, True, inst, n)); confirmed.add(m["id"])
done = True
break
except Exception:
with lock: attempts.append((ts, False, None, n))
time.sleep(0.2)
if not done:
with lock: unconfirmed += 1
threads = [threading.Thread(target=action)] + [threading.Thread(target=worker) for _ in range(WORKERS)]
[t.start() for t in threads]; [t.join() for t in threads]
stored, cur = set(), None
while True:
_, _, page = req("GET", path + "?limit=100" + (f"&cursor={cur}" if cur else ""), token=tok)
stored |= {m["id"] for m in page["items"]}
cur = page["next_cursor"]
if not cur: break
oks = sorted(a[0] for a in attempts if a[1])
gap = max((b - a for a, b in zip(oks, oks[1:])), default=0)
first_fail = [a[0] - STOP_AT for a in attempts if not a[1] and a[3] == 0 and a[0] >= STOP_AT]
pre_stop_errors = sum(1 for a in attempts if not a[1] and a[3] == 0 and a[0] < STOP_AT)
rps = lambda a, b: round(sum(a <= t < b for t in oks) / (b - a), 1)
late = {a[2] for a in attempts if a[1] and STOP_AT + 6 < a[0] < START_AT}
print(json.dumps({
"container": container, "stopped_instance": stopped_id,
"rps_before": rps(0, STOP_AT), "rps_during": rps(STOP_AT, START_AT), "rps_after": rps(START_AT, DURATION),
"first_attempt_errors": len(first_fail),
"error_window_s_after_stop": [round(min(first_fail), 2), round(max(first_fail), 2)] if first_fail else None,
"longest_gap_without_success_s": round(gap, 2),
"stopped_instance_seen_while_down": stopped_id in late,
"confirmed": len(confirmed), "unconfirmed_keys": unconfirmed, "stored": len(stored),
"lost_confirmed": len(confirmed - stored),
"stored_minus_confirmed": len(stored) - len(confirmed),
}, indent=2))