forked from mtusi2026/mtusi2026lab3
Общие PostgreSQL, Valkey и идемпотентность сообщений нужны, чтобы чтение и запись переживали остановку одной реплики.
326 lines
11 KiB
Python
326 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Нагрузка, поочерёдный отказ каждой реплики и rolling update. Токены не печатает."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
|
|
def run(cmd, env=None, check=True):
|
|
return subprocess.run(
|
|
cmd,
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=check,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def compose(*args):
|
|
return run(["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args])
|
|
|
|
|
|
def api_ids():
|
|
proc = compose("ps", "-q", "api")
|
|
return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
|
|
|
|
|
|
def inspect(cid, fmt):
|
|
return run(["docker", "inspect", "-f", fmt, cid]).stdout.strip()
|
|
|
|
|
|
def percentile(values, p):
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
index = max(0, int(round((len(ordered) - 1) * p)))
|
|
return round(ordered[index] * 1000, 3)
|
|
|
|
|
|
class Load:
|
|
def __init__(self, base, actor, chat, concurrency=4):
|
|
self.base = base
|
|
self.actor = actor
|
|
self.chat = chat
|
|
self.concurrency = concurrency
|
|
self.events = []
|
|
self._stop = threading.Event()
|
|
self._origin = time.monotonic()
|
|
self._pool = None
|
|
self._futures = []
|
|
|
|
def now(self):
|
|
return time.monotonic() - self._origin
|
|
|
|
def start(self):
|
|
def worker(worker_id):
|
|
local = Client(self.base, timeout=3)
|
|
index = 0
|
|
while not self._stop.is_set():
|
|
stamp = self.now()
|
|
try:
|
|
response = local.message(
|
|
self.actor,
|
|
self.chat,
|
|
"load-%s-%s-%s" % (worker_id, index, uuid.uuid4().hex[:8]),
|
|
)
|
|
self.events.append({
|
|
"ok": True,
|
|
"t": stamp,
|
|
"dt": self.now() - stamp,
|
|
"instance": response["headers"].get("X-Instance-Id"),
|
|
})
|
|
except Failure:
|
|
self.events.append({
|
|
"ok": False,
|
|
"t": stamp,
|
|
"dt": self.now() - stamp,
|
|
"instance": None,
|
|
})
|
|
index += 1
|
|
|
|
self._pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
|
self._futures = [self._pool.submit(worker, i) for i in range(self.concurrency)]
|
|
|
|
def stop(self):
|
|
self._stop.set()
|
|
for future in self._futures:
|
|
future.result()
|
|
self._pool.shutdown(wait=True)
|
|
|
|
def summary(self, start, end):
|
|
window = [item for item in self.events if start <= item["t"] < end]
|
|
ok = [item for item in window if item["ok"]]
|
|
bad = [item for item in window if not item["ok"]]
|
|
elapsed = max(end - start, 0.001)
|
|
return {
|
|
"seconds": round(end - start, 3),
|
|
"successful_requests": len(ok),
|
|
"failed_requests": len(bad),
|
|
"successful_rps": round(len(ok) / elapsed, 3),
|
|
"error_ratio": round(len(bad) / len(window), 6) if window else None,
|
|
"success_latency_ms": {
|
|
"p50": percentile([item["dt"] for item in ok], 0.50),
|
|
"p95": percentile([item["dt"] for item in ok], 0.95),
|
|
"p99": percentile([item["dt"] for item in ok], 0.99),
|
|
},
|
|
"instances": sorted({item["instance"] for item in ok if item["instance"]}),
|
|
}
|
|
|
|
|
|
def wait_ready(base, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
try:
|
|
Client(base, timeout=2).call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise SystemExit("API did not become ready")
|
|
time.sleep(0.5)
|
|
|
|
|
|
def wait_instance(base, actor, wanted, timeout):
|
|
started = time.monotonic()
|
|
seen = set()
|
|
while time.monotonic() - started < timeout:
|
|
try:
|
|
seen.add(
|
|
Client(base, timeout=2).call(
|
|
"GET", "/api/v1/users/me", actor=actor, schema="User"
|
|
)["headers"]["X-Instance-Id"]
|
|
)
|
|
except Failure:
|
|
pass
|
|
if wanted in seen:
|
|
return round(time.monotonic() - started, 3), sorted(seen)
|
|
time.sleep(0.2)
|
|
return None, sorted(seen)
|
|
|
|
|
|
def container_limits():
|
|
rows = []
|
|
proc = compose("ps", "-q")
|
|
for cid in [line.strip() for line in proc.stdout.splitlines() if line.strip()]:
|
|
rows.append({
|
|
"name": inspect(cid, "{{.Name}}").lstrip("/"),
|
|
"nano_cpus": int(inspect(cid, "{{.HostConfig.NanoCpus}}") or "0"),
|
|
"memory_bytes": int(inspect(cid, "{{.HostConfig.Memory}}") or "0"),
|
|
"published": inspect(cid, "{{json .NetworkSettings.Ports}}"),
|
|
})
|
|
return rows
|
|
|
|
|
|
def main():
|
|
base = os.environ.get("BASE_URL", "http://localhost:8080")
|
|
wait_ready(base)
|
|
client = Client(base, timeout=10)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
spread = [
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(30)
|
|
]
|
|
observed = sorted(set(spread))
|
|
if len(observed) < 2:
|
|
raise SystemExit("need two instance ids, saw %s" % observed)
|
|
|
|
key = "probe-" + uuid.uuid4().hex
|
|
text = "параллельный ключ " + key
|
|
barrier = threading.Barrier(8)
|
|
|
|
def once(_n):
|
|
local = Client(base, timeout=10)
|
|
barrier.wait()
|
|
response = local.message(alice, chat, text, key=key)
|
|
return response["json"]["id"], response["headers"]["X-Instance-Id"]
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
parallel = list(pool.map(once, range(8)))
|
|
parallel_ids = sorted({item[0] for item in parallel})
|
|
parallel_instances = sorted({item[1] for item in parallel})
|
|
if len(parallel_ids) != 1:
|
|
raise SystemExit("parallel idempotency created %s messages" % len(parallel_ids))
|
|
|
|
load = Load(base, alice, chat)
|
|
load.start()
|
|
time.sleep(8)
|
|
baseline_end = load.now()
|
|
baseline = load.summary(0, baseline_end)
|
|
|
|
failovers = []
|
|
for target in observed:
|
|
time.sleep(1)
|
|
mark = load.now()
|
|
env = os.environ.copy()
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
started = time.monotonic()
|
|
proc = run([str(ROOT / "scripts" / "stop-one.sh")], env=env, check=False)
|
|
stop_s = round(time.monotonic() - started, 3)
|
|
stopped_at = load.now()
|
|
time.sleep(6)
|
|
window_end = load.now()
|
|
after = [item for item in load.events if item["t"] >= stopped_at and item["ok"]]
|
|
served = sorted({item["instance"] for item in after if item["instance"]})
|
|
first = None
|
|
for item in load.events:
|
|
if item["t"] >= mark and item["ok"] and item["instance"] and item["instance"] != target:
|
|
first = round(item["t"] - mark, 3)
|
|
break
|
|
restore_started = time.monotonic()
|
|
run([str(ROOT / "scripts" / "restore-api.sh")])
|
|
back_s, seen_after = wait_instance(base, alice, target, 40)
|
|
failovers.append({
|
|
"target": target,
|
|
"stop_hook_exit": proc.returncode,
|
|
"stop_hook_seconds": stop_s,
|
|
"window": load.summary(mark, window_end),
|
|
"first_other_replica_success_seconds": first,
|
|
"target_seen_after_hook_returned": target in served,
|
|
"instances_after_hook": served,
|
|
"restore_until_target_seen_seconds": back_s,
|
|
"instances_after_restore": seen_after,
|
|
"restore_command_seconds": round(time.monotonic() - restore_started, 3),
|
|
})
|
|
if proc.returncode != 0:
|
|
load.stop()
|
|
raise SystemExit("stop-one failed for %s\n%s" % (target, proc.stderr))
|
|
if first is None or first > 10 or target in served:
|
|
load.stop()
|
|
raise SystemExit("failover criteria failed for %s" % target)
|
|
|
|
images_before = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
|
|
roll_mark = load.now()
|
|
holder = {}
|
|
|
|
def run_roll():
|
|
proc = run([str(ROOT / "scripts" / "rolling-update.sh")], check=False)
|
|
holder["code"] = proc.returncode
|
|
holder["err"] = (proc.stderr or "")[-800:]
|
|
holder["out"] = (proc.stdout or "")[-800:]
|
|
|
|
roller = threading.Thread(target=run_roll)
|
|
roller.start()
|
|
roller.join()
|
|
roll_end = load.now()
|
|
load.stop()
|
|
images_after = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
|
|
rolling = load.summary(roll_mark, roll_end)
|
|
rolling["exit"] = holder.get("code")
|
|
rolling["stderr_tail"] = holder.get("err")
|
|
if holder.get("code") != 0:
|
|
raise SystemExit("rolling update failed: %s\n%s" % (holder.get("err"), holder.get("out")))
|
|
|
|
report = {
|
|
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"platform": platform.platform(),
|
|
"cpu_count": os.cpu_count(),
|
|
"workload": "closed-loop POST, concurrency 4, client timeout 3s, no client retry",
|
|
"observed_instances": observed,
|
|
"parallel_idempotency": {
|
|
"requests": 8,
|
|
"unique_message_ids": len(parallel_ids),
|
|
"instances": parallel_instances,
|
|
},
|
|
"baseline": baseline,
|
|
"failovers": failovers,
|
|
"images_before_roll": images_before,
|
|
"images_after_roll": images_after,
|
|
"rolling_under_load": rolling,
|
|
"container_limits": container_limits(),
|
|
"lb_timeouts": {
|
|
"check_interval_seconds": 1,
|
|
"unhealthy_after_failures": 1,
|
|
"check_timeout_seconds": 1,
|
|
"max_detection_seconds": 2,
|
|
"connect_timeout_seconds": 1,
|
|
"server_timeout_seconds": 8,
|
|
"client_timeout_seconds": 8,
|
|
"drain_seconds": 3,
|
|
"stop_grace_seconds": 10,
|
|
},
|
|
}
|
|
dest = ROOT / "evidence" / "grade4.json"
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps({
|
|
"observed_instances": observed,
|
|
"parallel_unique_messages": len(parallel_ids),
|
|
"parallel_instances": parallel_instances,
|
|
"baseline_rps": baseline["successful_rps"],
|
|
"baseline_errors": baseline["failed_requests"],
|
|
"failovers": [
|
|
{
|
|
"target": item["target"],
|
|
"stop_s": item["stop_hook_seconds"],
|
|
"first_other_s": item["first_other_replica_success_seconds"],
|
|
"errors": item["window"]["failed_requests"],
|
|
"rps": item["window"]["successful_rps"],
|
|
"target_seen_after_hook": item["target_seen_after_hook_returned"],
|
|
"restore_s": item["restore_until_target_seen_seconds"],
|
|
}
|
|
for item in failovers
|
|
],
|
|
"rolling_exit": rolling["exit"],
|
|
"rolling_errors": rolling["failed_requests"],
|
|
"rolling_rps": rolling["successful_rps"],
|
|
"images_changed": images_before != images_after,
|
|
"evidence": "evidence/grade4.json",
|
|
}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|