Fix replica auth and idempotency races
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-5 track 1: repeatable 1/2/4 API-replica scaling experiment.
|
||||
|
||||
The script measures the same closed-loop write workload in two regimes:
|
||||
``fixed_total`` divides a fixed 2 CPU / 1 GiB API budget among replicas;
|
||||
``growing_total`` assigns every replica 1 CPU / 512 MiB. It writes raw output
|
||||
from tests/load.py to evidence/scale-*.json and a sanitized summary to
|
||||
evidence/scale-summary.json. No secrets or tokens are written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_COMPOSE = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
|
||||
SCALE_COMPOSE = BASE_COMPOSE + ["-f", "compose.scale.yaml"]
|
||||
|
||||
|
||||
def run(command: list[str], *, timeout: int = 180) -> None:
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
||||
if result.returncode:
|
||||
raise SystemExit(
|
||||
"command failed:\n"
|
||||
+ " ".join(command)
|
||||
+ "\n"
|
||||
+ (result.stdout + result.stderr)[-2000:]
|
||||
)
|
||||
|
||||
|
||||
def wait_ready(base_url: str) -> None:
|
||||
import urllib.request
|
||||
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base_url + "/health/ready", timeout=3) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("load balancer did not become ready within 120 seconds")
|
||||
|
||||
|
||||
def set_budget(replicas: int, regime: str) -> None:
|
||||
if regime == "fixed_total":
|
||||
cpu = 2.0 / replicas
|
||||
memory = 1024 // replicas
|
||||
else:
|
||||
cpu = 1.0
|
||||
memory = 512
|
||||
for index in range(1, replicas + 1):
|
||||
run([
|
||||
"docker", "update", "--cpus", str(cpu), "--memory", str(memory) + "m",
|
||||
"--memory-swap", str(memory) + "m",
|
||||
"mtusi2026lab3-api-" + str(index) + "-1",
|
||||
])
|
||||
|
||||
|
||||
def configure_replicas(replicas: int) -> None:
|
||||
if replicas in {1, 2}:
|
||||
# The standard two-replica topology remains intact; stop api-2 only for
|
||||
# the one-replica measurement. Nginx retries the healthy peer.
|
||||
run(BASE_COMPOSE + ["up", "-d", "--wait", "api-1", "api-2", "nginx"])
|
||||
if replicas == 1:
|
||||
run(BASE_COMPOSE + ["stop", "api-2"])
|
||||
else:
|
||||
run(SCALE_COMPOSE + ["up", "-d", "--wait", "api-1", "api-2", "api-3", "api-4", "nginx"])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8081"))
|
||||
parser.add_argument("--duration", type=float, default=15.0)
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.duration <= 300:
|
||||
raise SystemExit("duration must be 1..300")
|
||||
|
||||
evidence = ROOT / "evidence"
|
||||
evidence.mkdir(exist_ok=True)
|
||||
results: list[dict[str, object]] = []
|
||||
try:
|
||||
for regime in ("fixed_total", "growing_total"):
|
||||
for replicas in (1, 2, 4):
|
||||
configure_replicas(replicas)
|
||||
set_budget(replicas, regime)
|
||||
wait_ready(args.base_url.rstrip("/"))
|
||||
output = evidence / ("scale-" + regime + "-" + str(replicas) + ".json")
|
||||
run([
|
||||
"python", "tests/load.py", "--base-url", args.base_url,
|
||||
"--duration", str(args.duration), "--concurrency", str(args.concurrency),
|
||||
"--output", str(output),
|
||||
], timeout=int(args.duration + 120))
|
||||
sample = json.loads(output.read_text(encoding="utf-8"))
|
||||
sample.update({"regime": regime, "replicas": replicas})
|
||||
results.append(sample)
|
||||
finally:
|
||||
# Return to the normal grade-3 topology with its ordinary nginx config.
|
||||
run(SCALE_COMPOSE + ["down"], timeout=120)
|
||||
run(BASE_COMPOSE + ["up", "-d", "--wait"], timeout=180)
|
||||
|
||||
summary = {
|
||||
"method": "closed-loop POST messages; no retries; latency percentiles contain successes only",
|
||||
"fixed_total_budget": "2 CPU / 1024 MiB divided equally among 1, 2, 4 replicas",
|
||||
"growing_total_budget": "1 CPU / 512 MiB per replica",
|
||||
"results": results,
|
||||
"limitations": "Docker Desktop host scheduling and closed-loop coordinated omission; this is not a production capacity claim.",
|
||||
}
|
||||
(evidence / "scale-summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user