120 lines
5.6 KiB
Python
120 lines
5.6 KiB
Python
"""Run the unchanged public load generator during explicit API operations.
|
|
|
|
Usage: python -X utf8 student-tests/operations_load.py baseline|stop1|stop2|rolling
|
|
Only synthetic HTTP data are created. Volumes and shared stores are preserved.
|
|
"""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BASE = "http://127.0.0.1:8083"
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(["docker", *args], cwd=ROOT, check=True,
|
|
text=True, capture_output=True).stdout.strip()
|
|
|
|
|
|
def containers():
|
|
ids = docker("compose", "ps", "-q", "api").split()
|
|
return sorted(ids, key=lambda item: docker("inspect", "--format", "{{.Name}}", item))
|
|
|
|
|
|
def wait_visible(container):
|
|
deadline = time.monotonic() + 40
|
|
alias = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if alias is None:
|
|
alias = docker("exec", container, "/server", "instance-id")
|
|
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
|
|
if response.headers.get("X-Instance-Id") == alias:
|
|
return alias
|
|
except (subprocess.CalledProcessError, urllib.error.URLError, TimeoutError):
|
|
pass
|
|
time.sleep(0.2)
|
|
raise RuntimeError("API did not become visible through LB within 40 seconds")
|
|
|
|
|
|
def main():
|
|
mode = sys.argv[1]
|
|
if mode not in {"baseline", "stop1", "stop2", "rolling"}:
|
|
raise ValueError("Unknown mode")
|
|
original = containers()
|
|
if len(original) != 2:
|
|
raise RuntimeError("Exactly two running APIs required")
|
|
for container in original:
|
|
wait_visible(container)
|
|
duration = 50 if mode == "rolling" else 25
|
|
output = ROOT / "evidence" / ("load-" + mode + ".json")
|
|
log = ROOT / "evidence" / ("load-" + mode + ".txt")
|
|
operations = []
|
|
# Capture timestamps independently of load.py, whose elapsed interval starts
|
|
# after its own bootstrap. These are separate timing domains.
|
|
started = time.monotonic()
|
|
with log.open("w", encoding="utf-8") as stream:
|
|
process = subprocess.Popen([
|
|
sys.executable, "-X", "utf8", "tests/load.py", "--base-url", BASE,
|
|
"--duration", str(duration), "--concurrency", "4", "--output", str(output),
|
|
], cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT)
|
|
try:
|
|
time.sleep(5)
|
|
if process.poll() is not None:
|
|
raise RuntimeError("Load generator stopped before operation")
|
|
if mode.startswith("stop"):
|
|
target = original[int(mode[-1]) - 1]
|
|
action_started = time.monotonic()
|
|
docker("stop", "--time", "1", target)
|
|
# Probe API availability after Docker has confirmed the stop.
|
|
survivor = next(item for item in original if item != target)
|
|
wait_visible(survivor)
|
|
operations.append({"action": "stop", "container": target,
|
|
"offset_seconds": round(action_started - started, 3),
|
|
"stop_and_ready_seconds": round(time.monotonic() - action_started, 3)})
|
|
elif mode == "rolling":
|
|
for target in original:
|
|
survivor = next(item for item in containers() if item != target)
|
|
action_started = time.monotonic()
|
|
docker("stop", "--time", "15", target)
|
|
docker("rm", target)
|
|
# Keep the survivor intact, create only the missing replica.
|
|
docker("compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
|
|
current = containers()
|
|
if survivor not in current or target in current or len(current) != 2:
|
|
raise RuntimeError("Replacement changed survivor or did not replace target")
|
|
replacement = next(item for item in current if item != survivor)
|
|
alias = wait_visible(replacement)
|
|
operations.append({"action": "replace", "old_container": target,
|
|
"new_container": replacement, "new_instance_alias": alias,
|
|
"survivor_container": survivor,
|
|
"offset_seconds": round(action_started - started, 3),
|
|
"replacement_ready_seconds": round(time.monotonic() - action_started, 3)})
|
|
if process.poll() is not None:
|
|
raise RuntimeError("Load did not span both replacements")
|
|
time.sleep(3)
|
|
code = process.wait(timeout=duration + 20)
|
|
if code != 0:
|
|
raise RuntimeError("Load failed; inspect " + str(log))
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
docker("compose", "up", "-d", "--wait", "--wait-timeout", "60")
|
|
metadata = {
|
|
"mode": mode, "operations": operations,
|
|
"docker": json.loads(docker("info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":"{{.ServerVersion}}"}')),
|
|
"note": "One run per mode, concurrency 4, shared chat, no automatic POST retries; probes use GET ready. Replacement uses the same image and no migrations.",
|
|
}
|
|
(ROOT / "evidence" / ("operations-" + mode + ".json")).write_text(
|
|
json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
|
|
print(mode + ": PASS; metrics in " + str(output.relative_to(ROOT)), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|