Files
mtusi2026lab3/scripts/experiments.py

209 lines
9.7 KiB
Python

"""Repeatable API failures and sequential replacement under HTTP load."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import sys
import threading
import time
from uuid import uuid4
from ops import ROOT, command, compose, containers, summary
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def wait_instances(client, actor, expected, excluded=(), version=None):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
seen = {}
try:
for _ in range(20):
r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version")
if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen):
if version is None or all(v == version for v in seen.values()):
return seen
except Failure:
pass
time.sleep(.2)
raise AssertionError("Expected API replicas were not observed through the load balancer")
def history(client, actor, chat):
from urllib.parse import quote
result, cursor = [], None
while True:
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
if cursor:
path += "&cursor=" + quote(cursor, safe="")
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
result.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return result
def resource_sample():
ids = [r["id"] for r in summary() if r["status"] == "running"]
return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()]
def workload(base_url, label, action, duration=24, concurrency=4):
client = Client(base_url, timeout=3)
alice, bob, _, chat = client.bootstrap()
before_message = client.message(alice, chat, "confirmed before " + label)["json"]
observed_before = wait_instances(client, alice, 2)
started = time.monotonic()
deadline = started + duration
events, confirmations = [], {}
action_result = {}
lock = threading.Lock()
def worker(index):
local = Client(base_url, timeout=3)
key = uuid4().hex
attempt = 0
while time.monotonic() < deadline:
begin = time.monotonic()
try:
if index == 0 and attempt % 4 == 0:
response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
kind = "read"
else:
response = local.message(alice, chat, "load-" + key, key=key)
kind = "write"
with lock:
confirmations[response["json"]["id"]] = response["json"]
key = uuid4().hex
event = {"at": time.monotonic() - started, "ok": True, "kind": kind,
"ms": (time.monotonic() - begin) * 1000,
"instance": response["headers"]["X-Instance-Id"]}
except Failure:
event = {"at": time.monotonic() - started, "ok": False,
"ms": (time.monotonic() - begin) * 1000}
# Preserve the key after uncertain POST responses.
with lock:
events.append(event)
attempt += 1
def disrupt():
time.sleep(4)
action_result["started_seconds"] = time.monotonic() - started
action_result["detail"] = action(client, alice)
action_result["finished_seconds"] = time.monotonic() - started
action_result["resources_during_load"] = resource_sample()
with ThreadPoolExecutor(max_workers=concurrency + 1) as pool:
workers = [pool.submit(worker, i) for i in range(concurrency)]
change = pool.submit(disrupt)
for future in workers:
future.result()
change.result()
elapsed = time.monotonic() - started
rows = history(client, bob, chat)
by_id = {r["id"]: r for r in rows}
assert by_id[before_message["id"]] == before_message
assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost"
assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages"
success = [e for e in events if e["ok"]]
failure = [e for e in events if not e["ok"]]
finish = action_result["finished_seconds"]
writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"]
reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"]
assert writes_after and reads_after, "Read/write did not resume"
recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"]
if label.startswith("failure"):
assert recovery <= 10, "Failover exceeded 10 seconds"
hostname = action_result["detail"]["hostname"]
assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:])
latencies = sorted(e["ms"] for e in success)
def percentile(q):
return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3)
buckets = []
for start in range(0, math.ceil(elapsed), 2):
group = [e for e in events if start <= e["at"] < start + 2]
buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2,
"errors": sum(not e["ok"] for e in group)})
report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(),
"duration_seconds": round(elapsed, 3), "concurrency": concurrency,
"successful_requests": len(success), "failed_attempts": len(failure),
"error_ratio": round(len(failure) / len(events), 6),
"successful_rps": round(len(success) / elapsed, 3),
"successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
"first_write_after_action_seconds": round(recovery, 3), "action": action_result,
"confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows),
"unique_history_messages": len(by_id), "instances_before": list(observed_before),
"series_2_seconds": buckets, "deployment_after": summary(),
"note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."}
(ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n")
print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True)
return report
def failure_action(target):
def stop(client, actor):
begin = time.monotonic()
command(["docker", "stop", "--time", "6", target["Id"]], timeout=10)
row = json.loads(command(["docker", "inspect", target["Id"]]))[0]
assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0
elapsed = time.monotonic() - begin
assert elapsed <= 10
return {"container": row["Id"], "hostname": row["Config"]["Hostname"],
"stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]}
return stop
def rolling_action(client, actor):
original = containers()
assert len(original) == 2
updates = []
env = dict(os.environ, APP_VERSION="lab3-v2")
# Both migrations are additive. Check/replay once before replacing processes.
migration = compose("run", "--rm", "--no-deps", "migrate")
for old in original:
address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge"))
slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address)
command(["docker", "stop", "--time", "6", old["Id"]])
command(["docker", "rm", old["Id"]])
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env)
compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot)
current = containers()
fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}]
assert len(fresh) == 1
wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]])
updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot,
"new_image": fresh[0]["Image"]})
wait_instances(client, actor, 2, version="lab3-v2")
return {"updates": updates, "migration": migration,
"method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"}
def main():
import argparse
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
originals = containers()
assert len(originals) == 2
reports = []
for index, target in enumerate(originals, 1):
try:
reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target)))
finally:
command(["docker", "start", target["Id"]])
c = Client(args.base_url)
alice = c.user("restore")
wait_instances(c, alice, 2)
reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45))
(ROOT / "evidence" / "experiments.json").write_text(json.dumps([
{k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds",
"confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n")
if __name__ == "__main__":
main()