"""Repeatable API failures and sequential replacement under HTTP load.""" from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import json import math import os from pathlib import Path import sys import threading import time from uuid import uuid4 from ops import ROOT, command, compose, containers, summary sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure def wait_instances(client, actor, expected, excluded=(), version=None): deadline = time.monotonic() + 60 while time.monotonic() < deadline: seen = {} try: for _ in range(20): r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User") seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version") if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen): if version is None or all(v == version for v in seen.values()): return seen except Failure: pass time.sleep(.2) raise AssertionError("Expected API replicas were not observed through the load balancer") def history(client, actor, chat): from urllib.parse import quote result, cursor = [], None while True: path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" if cursor: path += "&cursor=" + quote(cursor, safe="") page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] result.extend(page["items"]) cursor = page["next_cursor"] if cursor is None: return result def resource_sample(): ids = [r["id"] for r in summary() if r["status"] == "running"] return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()] def workload(base_url, label, action, duration=24, concurrency=4): client = Client(base_url, timeout=3) alice, bob, _, chat = client.bootstrap() before_message = client.message(alice, chat, "confirmed before " + label)["json"] observed_before = wait_instances(client, alice, 2) started = time.monotonic() deadline = started + duration events, confirmations = [], {} action_result = {} lock = threading.Lock() def worker(index): local = Client(base_url, timeout=3) key = uuid4().hex attempt = 0 while time.monotonic() < deadline: begin = time.monotonic() try: if index == 0 and attempt % 4 == 0: response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User") kind = "read" else: response = local.message(alice, chat, "load-" + key, key=key) kind = "write" with lock: confirmations[response["json"]["id"]] = response["json"] key = uuid4().hex event = {"at": time.monotonic() - started, "ok": True, "kind": kind, "ms": (time.monotonic() - begin) * 1000, "instance": response["headers"]["X-Instance-Id"]} except Failure: event = {"at": time.monotonic() - started, "ok": False, "ms": (time.monotonic() - begin) * 1000} # Preserve the key after uncertain POST responses. with lock: events.append(event) attempt += 1 def disrupt(): time.sleep(4) action_result["started_seconds"] = time.monotonic() - started action_result["detail"] = action(client, alice) action_result["finished_seconds"] = time.monotonic() - started action_result["resources_during_load"] = resource_sample() with ThreadPoolExecutor(max_workers=concurrency + 1) as pool: workers = [pool.submit(worker, i) for i in range(concurrency)] change = pool.submit(disrupt) for future in workers: future.result() change.result() elapsed = time.monotonic() - started rows = history(client, bob, chat) by_id = {r["id"]: r for r in rows} assert by_id[before_message["id"]] == before_message assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost" assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages" success = [e for e in events if e["ok"]] failure = [e for e in events if not e["ok"]] finish = action_result["finished_seconds"] writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"] reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"] assert writes_after and reads_after, "Read/write did not resume" recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"] if label.startswith("failure"): assert recovery <= 10, "Failover exceeded 10 seconds" hostname = action_result["detail"]["hostname"] assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:]) latencies = sorted(e["ms"] for e in success) def percentile(q): return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3) buckets = [] for start in range(0, math.ceil(elapsed), 2): group = [e for e in events if start <= e["at"] < start + 2] buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2, "errors": sum(not e["ok"] for e in group)}) report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(), "duration_seconds": round(elapsed, 3), "concurrency": concurrency, "successful_requests": len(success), "failed_attempts": len(failure), "error_ratio": round(len(failure) / len(events), 6), "successful_rps": round(len(success) / elapsed, 3), "successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)}, "first_write_after_action_seconds": round(recovery, 3), "action": action_result, "confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows), "unique_history_messages": len(by_id), "instances_before": list(observed_before), "series_2_seconds": buckets, "deployment_after": summary(), "note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."} (ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n") print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True) return report def failure_action(target): def stop(client, actor): begin = time.monotonic() command(["docker", "stop", "--time", "6", target["Id"]], timeout=10) row = json.loads(command(["docker", "inspect", target["Id"]]))[0] assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0 elapsed = time.monotonic() - begin assert elapsed <= 10 return {"container": row["Id"], "hostname": row["Config"]["Hostname"], "stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]} return stop def rolling_action(client, actor): original = containers() assert len(original) == 2 updates = [] env = dict(os.environ, APP_VERSION="lab3-v2") # Both migrations are additive. Check/replay once before replacing processes. migration = compose("run", "--rm", "--no-deps", "migrate") for old in original: address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge")) slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address) command(["docker", "stop", "--time", "6", old["Id"]]) command(["docker", "rm", old["Id"]]) compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env) compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot) current = containers() fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}] assert len(fresh) == 1 wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]]) updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot, "new_image": fresh[0]["Image"]}) wait_instances(client, actor, 2, version="lab3-v2") return {"updates": updates, "migration": migration, "method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"} def main(): import argparse p = argparse.ArgumentParser() p.add_argument("--base-url", default="http://127.0.0.1:8080") args = p.parse_args() originals = containers() assert len(originals) == 2 reports = [] for index, target in enumerate(originals, 1): try: reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target))) finally: command(["docker", "start", target["Id"]]) c = Client(args.base_url) alice = c.user("restore") wait_instances(c, alice, 2) reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45)) (ROOT / "evidence" / "experiments.json").write_text(json.dumps([ {k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds", "confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n") if __name__ == "__main__": main()