209 lines
9.7 KiB
Python
209 lines
9.7 KiB
Python
"""Repeatable API failures and sequential replacement under HTTP load."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import threading
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
from ops import ROOT, command, compose, containers, summary
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
|
|
def wait_instances(client, actor, expected, excluded=(), version=None):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
seen = {}
|
|
try:
|
|
for _ in range(20):
|
|
r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version")
|
|
if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen):
|
|
if version is None or all(v == version for v in seen.values()):
|
|
return seen
|
|
except Failure:
|
|
pass
|
|
time.sleep(.2)
|
|
raise AssertionError("Expected API replicas were not observed through the load balancer")
|
|
|
|
|
|
def history(client, actor, chat):
|
|
from urllib.parse import quote
|
|
result, cursor = [], None
|
|
while True:
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
|
if cursor:
|
|
path += "&cursor=" + quote(cursor, safe="")
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
result.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
return result
|
|
|
|
|
|
def resource_sample():
|
|
ids = [r["id"] for r in summary() if r["status"] == "running"]
|
|
return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()]
|
|
|
|
|
|
def workload(base_url, label, action, duration=24, concurrency=4):
|
|
client = Client(base_url, timeout=3)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
before_message = client.message(alice, chat, "confirmed before " + label)["json"]
|
|
observed_before = wait_instances(client, alice, 2)
|
|
started = time.monotonic()
|
|
deadline = started + duration
|
|
events, confirmations = [], {}
|
|
action_result = {}
|
|
lock = threading.Lock()
|
|
|
|
def worker(index):
|
|
local = Client(base_url, timeout=3)
|
|
key = uuid4().hex
|
|
attempt = 0
|
|
while time.monotonic() < deadline:
|
|
begin = time.monotonic()
|
|
try:
|
|
if index == 0 and attempt % 4 == 0:
|
|
response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
kind = "read"
|
|
else:
|
|
response = local.message(alice, chat, "load-" + key, key=key)
|
|
kind = "write"
|
|
with lock:
|
|
confirmations[response["json"]["id"]] = response["json"]
|
|
key = uuid4().hex
|
|
event = {"at": time.monotonic() - started, "ok": True, "kind": kind,
|
|
"ms": (time.monotonic() - begin) * 1000,
|
|
"instance": response["headers"]["X-Instance-Id"]}
|
|
except Failure:
|
|
event = {"at": time.monotonic() - started, "ok": False,
|
|
"ms": (time.monotonic() - begin) * 1000}
|
|
# Preserve the key after uncertain POST responses.
|
|
with lock:
|
|
events.append(event)
|
|
attempt += 1
|
|
|
|
def disrupt():
|
|
time.sleep(4)
|
|
action_result["started_seconds"] = time.monotonic() - started
|
|
action_result["detail"] = action(client, alice)
|
|
action_result["finished_seconds"] = time.monotonic() - started
|
|
action_result["resources_during_load"] = resource_sample()
|
|
|
|
with ThreadPoolExecutor(max_workers=concurrency + 1) as pool:
|
|
workers = [pool.submit(worker, i) for i in range(concurrency)]
|
|
change = pool.submit(disrupt)
|
|
for future in workers:
|
|
future.result()
|
|
change.result()
|
|
elapsed = time.monotonic() - started
|
|
rows = history(client, bob, chat)
|
|
by_id = {r["id"]: r for r in rows}
|
|
assert by_id[before_message["id"]] == before_message
|
|
assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost"
|
|
assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages"
|
|
success = [e for e in events if e["ok"]]
|
|
failure = [e for e in events if not e["ok"]]
|
|
finish = action_result["finished_seconds"]
|
|
writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"]
|
|
reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"]
|
|
assert writes_after and reads_after, "Read/write did not resume"
|
|
recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"]
|
|
if label.startswith("failure"):
|
|
assert recovery <= 10, "Failover exceeded 10 seconds"
|
|
hostname = action_result["detail"]["hostname"]
|
|
assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:])
|
|
latencies = sorted(e["ms"] for e in success)
|
|
def percentile(q):
|
|
return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3)
|
|
buckets = []
|
|
for start in range(0, math.ceil(elapsed), 2):
|
|
group = [e for e in events if start <= e["at"] < start + 2]
|
|
buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2,
|
|
"errors": sum(not e["ok"] for e in group)})
|
|
report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(),
|
|
"duration_seconds": round(elapsed, 3), "concurrency": concurrency,
|
|
"successful_requests": len(success), "failed_attempts": len(failure),
|
|
"error_ratio": round(len(failure) / len(events), 6),
|
|
"successful_rps": round(len(success) / elapsed, 3),
|
|
"successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
|
|
"first_write_after_action_seconds": round(recovery, 3), "action": action_result,
|
|
"confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows),
|
|
"unique_history_messages": len(by_id), "instances_before": list(observed_before),
|
|
"series_2_seconds": buckets, "deployment_after": summary(),
|
|
"note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."}
|
|
(ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n")
|
|
print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True)
|
|
return report
|
|
|
|
|
|
def failure_action(target):
|
|
def stop(client, actor):
|
|
begin = time.monotonic()
|
|
command(["docker", "stop", "--time", "6", target["Id"]], timeout=10)
|
|
row = json.loads(command(["docker", "inspect", target["Id"]]))[0]
|
|
assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0
|
|
elapsed = time.monotonic() - begin
|
|
assert elapsed <= 10
|
|
return {"container": row["Id"], "hostname": row["Config"]["Hostname"],
|
|
"stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]}
|
|
return stop
|
|
|
|
|
|
def rolling_action(client, actor):
|
|
original = containers()
|
|
assert len(original) == 2
|
|
updates = []
|
|
env = dict(os.environ, APP_VERSION="lab3-v2")
|
|
# Both migrations are additive. Check/replay once before replacing processes.
|
|
migration = compose("run", "--rm", "--no-deps", "migrate")
|
|
for old in original:
|
|
address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge"))
|
|
slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address)
|
|
command(["docker", "stop", "--time", "6", old["Id"]])
|
|
command(["docker", "rm", old["Id"]])
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env)
|
|
compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot)
|
|
current = containers()
|
|
fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}]
|
|
assert len(fresh) == 1
|
|
wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]])
|
|
updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot,
|
|
"new_image": fresh[0]["Image"]})
|
|
wait_instances(client, actor, 2, version="lab3-v2")
|
|
return {"updates": updates, "migration": migration,
|
|
"method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"}
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
args = p.parse_args()
|
|
originals = containers()
|
|
assert len(originals) == 2
|
|
reports = []
|
|
for index, target in enumerate(originals, 1):
|
|
try:
|
|
reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target)))
|
|
finally:
|
|
command(["docker", "start", target["Id"]])
|
|
c = Client(args.base_url)
|
|
alice = c.user("restore")
|
|
wait_instances(c, alice, 2)
|
|
reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45))
|
|
(ROOT / "evidence" / "experiments.json").write_text(json.dumps([
|
|
{k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds",
|
|
"confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|