193 lines
9.7 KiB
Python
193 lines
9.7 KiB
Python
"""Run the unchanged public load generator during explicit API operations.
|
|
|
|
Usage: python -X utf8 student-tests/operations_load.py baseline|stop1|stop2|rolling
|
|
Only synthetic HTTP data are created. Volumes and shared stores are preserved.
|
|
"""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BASE = "http://127.0.0.1:8084"
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(["docker", *args], cwd=ROOT, check=True,
|
|
text=True, capture_output=True).stdout.strip()
|
|
|
|
|
|
def containers():
|
|
ids = docker("compose", "ps", "-q", "api").split()
|
|
return sorted(ids, key=lambda item: docker("inspect", "--format", "{{.Name}}", item))
|
|
|
|
|
|
def wait_visible(container):
|
|
deadline = time.monotonic() + 40
|
|
alias = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if alias is None:
|
|
alias = docker("exec", container, "/server", "instance-id")
|
|
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
|
|
if response.headers.get("X-Instance-Id") == alias:
|
|
return alias
|
|
except (subprocess.CalledProcessError, urllib.error.URLError, TimeoutError):
|
|
pass
|
|
time.sleep(0.2)
|
|
raise RuntimeError("API did not become visible through LB within 40 seconds")
|
|
|
|
|
|
def check_data(client, actor, chat, confirmed, action_started):
|
|
"""Verify real history reads and writes after stop, not just health."""
|
|
deadline = action_started + 10
|
|
key = uuid.uuid4().hex
|
|
first_read = first_write = None
|
|
message = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
|
actor=actor, schema="MessagePage")["json"]["items"]
|
|
if any(item not in history for item in confirmed):
|
|
raise RuntimeError("Confirmed probe message lost")
|
|
if first_read is None:
|
|
first_read = time.monotonic() - action_started
|
|
except Failure:
|
|
pass
|
|
if message is None:
|
|
try:
|
|
message = client.message(actor, chat, "operation-probe-" + key, key=key)["json"]
|
|
first_write = time.monotonic() - action_started
|
|
except Failure:
|
|
pass
|
|
if first_read is not None and message is not None:
|
|
confirmed.append(message)
|
|
return {"read_recovery_seconds": round(first_read, 3),
|
|
"write_recovery_seconds": round(first_write, 3)}
|
|
time.sleep(0.1)
|
|
raise RuntimeError("History/read or message/write did not recover within 10 seconds")
|
|
|
|
|
|
def main():
|
|
global BASE
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=["baseline", "stop1", "stop2", "rolling"])
|
|
parser.add_argument("--base-url", default=BASE)
|
|
parser.add_argument("--output-dir", default="evidence")
|
|
parser.add_argument("--force-address-change", action="store_true",
|
|
help="During rolling, briefly reserve the removed IP to verify Docker DNS updates")
|
|
args = parser.parse_args()
|
|
mode = args.mode
|
|
BASE = args.base_url.rstrip("/")
|
|
evidence_dir = ROOT / args.output_dir
|
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
original = containers()
|
|
if len(original) != 2:
|
|
raise RuntimeError("Exactly two running APIs required")
|
|
for container in original:
|
|
wait_visible(container)
|
|
probe = Client(BASE, timeout=1)
|
|
actor, _, _, probe_chat = probe.bootstrap()
|
|
confirmed = [probe.message(actor, probe_chat, "confirmed-before-operation")["json"]]
|
|
duration = 50 if mode == "rolling" else 25
|
|
output = evidence_dir / ("load-" + mode + ".json")
|
|
log = evidence_dir / ("load-" + mode + ".txt")
|
|
operations = []
|
|
# Capture timestamps independently of load.py, whose elapsed interval starts
|
|
# after its own bootstrap. These are separate timing domains.
|
|
started = time.monotonic()
|
|
with log.open("w", encoding="utf-8") as stream:
|
|
process = subprocess.Popen([
|
|
sys.executable, "-X", "utf8", "tests/load.py", "--base-url", BASE,
|
|
"--duration", str(duration), "--concurrency", "4", "--output", str(output),
|
|
], cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT)
|
|
try:
|
|
time.sleep(5)
|
|
if process.poll() is not None:
|
|
raise RuntimeError("Load generator stopped before operation")
|
|
if mode.startswith("stop"):
|
|
target = original[int(mode[-1]) - 1]
|
|
action_started = time.monotonic()
|
|
docker("stop", "--time", "1", target)
|
|
recovery = check_data(probe, actor, probe_chat, confirmed, action_started)
|
|
# Probe API availability after Docker has confirmed the stop.
|
|
survivor = next(item for item in original if item != target)
|
|
wait_visible(survivor)
|
|
operations.append({"action": "stop", "container": target,
|
|
"offset_seconds": round(action_started - started, 3),
|
|
"stop_and_ready_seconds": round(time.monotonic() - action_started, 3),
|
|
**recovery})
|
|
elif mode == "rolling":
|
|
for target in original:
|
|
survivor = next(item for item in containers() if item != target)
|
|
networks = json.loads(docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", target))
|
|
network_name, network = next(iter(networks.items()))
|
|
old_ip = network["IPAddress"]
|
|
action_started = time.monotonic()
|
|
docker("stop", "--time", "15", target)
|
|
recovery = check_data(probe, actor, probe_chat, confirmed, action_started)
|
|
docker("rm", target)
|
|
holder = None
|
|
try:
|
|
if args.force_address_change:
|
|
holder = docker("run", "--rm", "-d", "--network", network_name,
|
|
"--ip", old_ip, "--cpus", "0.05", "--memory", "16m",
|
|
"--entrypoint", "/bin/sh", "redis:7.4.11-alpine", "-c", "sleep 120")
|
|
# Keep the survivor intact, create only the missing replica.
|
|
docker("compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
|
|
current = containers()
|
|
if survivor not in current or target in current or len(current) != 2:
|
|
raise RuntimeError("Replacement changed survivor or did not replace target")
|
|
replacement = next(item for item in current if item != survivor)
|
|
new_networks = json.loads(docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", replacement))
|
|
new_ip = new_networks[network_name]["IPAddress"]
|
|
if args.force_address_change and new_ip == old_ip:
|
|
raise RuntimeError("Replacement did not change IP")
|
|
alias = wait_visible(replacement)
|
|
finally:
|
|
if holder is not None:
|
|
docker("rm", "-f", holder)
|
|
operations.append({"action": "replace", "old_container": target,
|
|
"new_container": replacement, "new_instance_alias": alias,
|
|
"survivor_container": survivor,
|
|
"old_ip": old_ip, "new_ip": new_ip,
|
|
"offset_seconds": round(action_started - started, 3),
|
|
"replacement_ready_seconds": round(time.monotonic() - action_started, 3),
|
|
**recovery})
|
|
if process.poll() is not None:
|
|
raise RuntimeError("Load did not span both replacements")
|
|
time.sleep(3)
|
|
code = process.wait(timeout=duration + 20)
|
|
if code != 0:
|
|
raise RuntimeError("Load failed; inspect " + str(log))
|
|
history = probe.call("GET", "/api/v1/chats/" + probe_chat["id"] + "/messages?limit=100",
|
|
actor=actor, schema="MessagePage")["json"]["items"]
|
|
if len(history) != len(confirmed) or any(item not in history for item in confirmed):
|
|
raise RuntimeError("Final probe history lost messages or contains duplicates")
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
docker("compose", "up", "-d", "--wait", "--wait-timeout", "60")
|
|
metadata = {
|
|
"mode": mode, "operations": operations,
|
|
"docker": json.loads(docker("info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":"{{.ServerVersion}}"}')),
|
|
"confirmed_probe_messages": len(confirmed),
|
|
"forced_address_change": args.force_address_change,
|
|
"note": "One run per mode, concurrency 4, shared load chat, no load POST retries; separate history/write probes retry with a stable key and validate confirmed data. Replacement uses the same image and no migrations.",
|
|
}
|
|
(evidence_dir / ("operations-" + mode + ".json")).write_text(
|
|
json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
|
|
print(mode + ": PASS; metrics in " + str(output.relative_to(ROOT)), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|