Files
mtusi2026lab4/student-tests/operations_load.py
T

193 lines
9.7 KiB
Python

"""Run the unchanged public load generator during explicit API operations.
Usage: python -X utf8 student-tests/operations_load.py baseline|stop1|stop2|rolling
Only synthetic HTTP data are created. Volumes and shared stores are preserved.
"""
import argparse
import json
from pathlib import Path
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parent.parent
BASE = "http://127.0.0.1:8084"
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def docker(*args):
return subprocess.run(["docker", *args], cwd=ROOT, check=True,
text=True, capture_output=True).stdout.strip()
def containers():
ids = docker("compose", "ps", "-q", "api").split()
return sorted(ids, key=lambda item: docker("inspect", "--format", "{{.Name}}", item))
def wait_visible(container):
deadline = time.monotonic() + 40
alias = None
while time.monotonic() < deadline:
try:
if alias is None:
alias = docker("exec", container, "/server", "instance-id")
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
if response.headers.get("X-Instance-Id") == alias:
return alias
except (subprocess.CalledProcessError, urllib.error.URLError, TimeoutError):
pass
time.sleep(0.2)
raise RuntimeError("API did not become visible through LB within 40 seconds")
def check_data(client, actor, chat, confirmed, action_started):
"""Verify real history reads and writes after stop, not just health."""
deadline = action_started + 10
key = uuid.uuid4().hex
first_read = first_write = None
message = None
while time.monotonic() < deadline:
try:
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=actor, schema="MessagePage")["json"]["items"]
if any(item not in history for item in confirmed):
raise RuntimeError("Confirmed probe message lost")
if first_read is None:
first_read = time.monotonic() - action_started
except Failure:
pass
if message is None:
try:
message = client.message(actor, chat, "operation-probe-" + key, key=key)["json"]
first_write = time.monotonic() - action_started
except Failure:
pass
if first_read is not None and message is not None:
confirmed.append(message)
return {"read_recovery_seconds": round(first_read, 3),
"write_recovery_seconds": round(first_write, 3)}
time.sleep(0.1)
raise RuntimeError("History/read or message/write did not recover within 10 seconds")
def main():
global BASE
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=["baseline", "stop1", "stop2", "rolling"])
parser.add_argument("--base-url", default=BASE)
parser.add_argument("--output-dir", default="evidence")
parser.add_argument("--force-address-change", action="store_true",
help="During rolling, briefly reserve the removed IP to verify Docker DNS updates")
args = parser.parse_args()
mode = args.mode
BASE = args.base_url.rstrip("/")
evidence_dir = ROOT / args.output_dir
evidence_dir.mkdir(parents=True, exist_ok=True)
original = containers()
if len(original) != 2:
raise RuntimeError("Exactly two running APIs required")
for container in original:
wait_visible(container)
probe = Client(BASE, timeout=1)
actor, _, _, probe_chat = probe.bootstrap()
confirmed = [probe.message(actor, probe_chat, "confirmed-before-operation")["json"]]
duration = 50 if mode == "rolling" else 25
output = evidence_dir / ("load-" + mode + ".json")
log = evidence_dir / ("load-" + mode + ".txt")
operations = []
# Capture timestamps independently of load.py, whose elapsed interval starts
# after its own bootstrap. These are separate timing domains.
started = time.monotonic()
with log.open("w", encoding="utf-8") as stream:
process = subprocess.Popen([
sys.executable, "-X", "utf8", "tests/load.py", "--base-url", BASE,
"--duration", str(duration), "--concurrency", "4", "--output", str(output),
], cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT)
try:
time.sleep(5)
if process.poll() is not None:
raise RuntimeError("Load generator stopped before operation")
if mode.startswith("stop"):
target = original[int(mode[-1]) - 1]
action_started = time.monotonic()
docker("stop", "--time", "1", target)
recovery = check_data(probe, actor, probe_chat, confirmed, action_started)
# Probe API availability after Docker has confirmed the stop.
survivor = next(item for item in original if item != target)
wait_visible(survivor)
operations.append({"action": "stop", "container": target,
"offset_seconds": round(action_started - started, 3),
"stop_and_ready_seconds": round(time.monotonic() - action_started, 3),
**recovery})
elif mode == "rolling":
for target in original:
survivor = next(item for item in containers() if item != target)
networks = json.loads(docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", target))
network_name, network = next(iter(networks.items()))
old_ip = network["IPAddress"]
action_started = time.monotonic()
docker("stop", "--time", "15", target)
recovery = check_data(probe, actor, probe_chat, confirmed, action_started)
docker("rm", target)
holder = None
try:
if args.force_address_change:
holder = docker("run", "--rm", "-d", "--network", network_name,
"--ip", old_ip, "--cpus", "0.05", "--memory", "16m",
"--entrypoint", "/bin/sh", "redis:7.4.11-alpine", "-c", "sleep 120")
# Keep the survivor intact, create only the missing replica.
docker("compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
current = containers()
if survivor not in current or target in current or len(current) != 2:
raise RuntimeError("Replacement changed survivor or did not replace target")
replacement = next(item for item in current if item != survivor)
new_networks = json.loads(docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", replacement))
new_ip = new_networks[network_name]["IPAddress"]
if args.force_address_change and new_ip == old_ip:
raise RuntimeError("Replacement did not change IP")
alias = wait_visible(replacement)
finally:
if holder is not None:
docker("rm", "-f", holder)
operations.append({"action": "replace", "old_container": target,
"new_container": replacement, "new_instance_alias": alias,
"survivor_container": survivor,
"old_ip": old_ip, "new_ip": new_ip,
"offset_seconds": round(action_started - started, 3),
"replacement_ready_seconds": round(time.monotonic() - action_started, 3),
**recovery})
if process.poll() is not None:
raise RuntimeError("Load did not span both replacements")
time.sleep(3)
code = process.wait(timeout=duration + 20)
if code != 0:
raise RuntimeError("Load failed; inspect " + str(log))
history = probe.call("GET", "/api/v1/chats/" + probe_chat["id"] + "/messages?limit=100",
actor=actor, schema="MessagePage")["json"]["items"]
if len(history) != len(confirmed) or any(item not in history for item in confirmed):
raise RuntimeError("Final probe history lost messages or contains duplicates")
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=10)
docker("compose", "up", "-d", "--wait", "--wait-timeout", "60")
metadata = {
"mode": mode, "operations": operations,
"docker": json.loads(docker("info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":"{{.ServerVersion}}"}')),
"confirmed_probe_messages": len(confirmed),
"forced_address_change": args.force_address_change,
"note": "One run per mode, concurrency 4, shared load chat, no load POST retries; separate history/write probes retry with a stable key and validate confirmed data. Replacement uses the same image and no migrations.",
}
(evidence_dir / ("operations-" + mode + ".json")).write_text(
json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
print(mode + ": PASS; metrics in " + str(output.relative_to(ROOT)), flush=True)
if __name__ == "__main__":
main()