Files
mtusi2026lab3/scripts/check_grade4.py
T

176 lines
8.4 KiB
Python

#!/usr/bin/env python3
"""Stops each API separately and replaces both under a small closed-loop load."""
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import sys
import subprocess
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
from resilience import history
import ops
def exercise(c, name, action):
alice, bob, eve, chat = c.bootstrap()
initial_key = uuid.uuid4().hex
initial = c.message(alice, chat, "До воздействия", key=initial_key)["json"]
observed = {c.call("GET", "/api/v1/users/me", actor=alice)["headers"]["X-Instance-Id"]
for _ in range(20)}
check(len(observed) >= 2, "need two actual instances before " + name)
stop = threading.Event()
barrier = threading.Barrier(5)
def worker(number):
local = Client(c.base, timeout=2)
writes, cycles, errors = [], [], 0
barrier.wait()
key = uuid.uuid4().hex
body = "worker-%d-%s" % (number, key)
while not stop.is_set():
started = time.monotonic()
try:
response = local.message(alice, chat, body, key=key)
writes.append(response["json"])
# Read with the original session while another backend may serve it.
local.call("GET", "/api/v1/chats/" + chat["id"], actor=bob, schema="Chat")
finished = time.monotonic()
cycles.append((finished, finished - started,
response["headers"]["X-Instance-Id"]))
key = uuid.uuid4().hex
body = "worker-%d-%s" % (number, key)
except Failure:
errors += 1
# An uncertain write keeps its original key and body.
time.sleep(0.05)
return writes, cycles, errors
started = time.monotonic()
with ThreadPoolExecutor(max_workers=4) as pool:
futures = [pool.submit(worker, n) for n in range(4)]
barrier.wait()
try:
time.sleep(1)
action_started = time.monotonic()
phases = action()
action_finished = time.monotonic()
time.sleep(7)
finally:
stop.set()
results = [f.result() for f in futures]
elapsed = time.monotonic() - started
writes = [msg for messages, _, _ in results for msg in messages]
cycles = sorted(item for _, values, _ in results for item in values)
errors = sum(n for _, _, n in results)
# An in-flight POST can finish before the stop and its paired GET after it.
# Only cycles started after the action count as traffic to a surviving API.
recovered = [t for t, duration, _ in cycles if t - duration >= action_finished]
check(bool(recovered), "no reads/writes after " + name)
recovery = recovered[0] - action_started
check(recovery <= 10 if name.startswith("stop-") else True,
"failover exceeded 10 seconds")
times = [started] + [t for t, _, _ in cycles] + [time.monotonic()]
max_gap = max(b - a for a, b in zip(times, times[1:]))
check(max_gap <= 10, "no successful read/write cycle for more than 10 seconds")
during = [t for t, _, _ in cycles if action_started <= t <= action_finished]
if name == "rolling-update":
check(bool(during), "no successful traffic during rolling replacement")
for phase in phases or []:
check(any(phase["started"] <= t <= phase["finished"] for t in during),
"no successful traffic while replacing " + phase["service"])
stored = history(c, bob, chat)
ids = {m["id"] for m in stored}
check(len(ids) == len(stored), "duplicate message IDs")
check(len({m["text"] for m in stored}) == len(stored), "retry created duplicate text")
by_id = {m["id"]: m for m in stored}
check(all(by_id.get(m["id"]) == m for m in [initial] + writes), "confirmed data lost")
check(c.message(alice, chat, initial["text"], key=initial_key)["json"] == initial,
"pre-action idempotency result lost")
latency = sorted(d * 1000 for _, d, _ in cycles)
def percentile(p):
return round(latency[max(0, math.ceil(p * len(latency)) - 1)], 3)
result = {"scenario": name, "concurrency": 4, "elapsed_seconds": round(elapsed, 3),
"successful_read_write_cycles": len(cycles), "failed_attempts": errors,
"successful_http_requests": len(writes) + len(cycles),
"successful_rps": round((len(writes) + len(cycles)) / elapsed, 3),
"successful_cycles_per_second": round(len(cycles) / elapsed, 3),
"successful_cycles_during_action": len(during),
"max_success_gap_seconds": round(max_gap, 3),
"first_success_after_action_seconds": round(recovery, 3),
"latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
"instances_before": sorted(observed),
"instances_after": sorted({i for t, duration, i in cycles
if t - duration >= action_finished}),
"confirmed_messages_preserved": len({m["id"] for m in [initial] + writes}),
"note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration."}
print(json.dumps(result, ensure_ascii=False, indent=2), flush=True)
return result
def main():
args = arguments("Lab3 grade4 Compose experiments")
c = Client(args.base_url, args.ca_file)
ops.BASE_URL = c.base
dest = ROOT / "evidence"
dest.mkdir(exist_ok=True)
http_args = ["--base-url", c.base]
if args.ca_file:
http_args += ["--ca-file", args.ca_file]
def checked(script, filename, *extra):
result = subprocess.run([sys.executable, str(ROOT / script), *extra], cwd=ROOT,
text=True, encoding="utf-8", stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=dict(os.environ, PYTHONIOENCODING="utf-8"))
(dest / filename).write_text(result.stdout, encoding="utf-8")
print(result.stdout, end="", flush=True)
check(result.returncode == 0, script + " failed; see evidence/" + filename)
ops.restore()
checked("scripts/check_contract.py", "check.txt")
checked("tests/smoke.py", "smoke.txt", *http_args)
checked("student-tests/run.py", "student-tests.txt", *http_args)
results = []
processes = []
for service in ops.SERVICES:
ops.restore()
before = ops.state(service)
try:
result = exercise(c, "stop-" + service, lambda: ops.stop_one(service))
after = ops.state(service)
check(after["status"] == "exited", "stopped API restarted unexpectedly")
check(all(i.split(":", 1)[0] != before["hostname"] for i in result["instances_after"]),
"stopped replica still handles requests")
results.append(result)
processes.append({"action": "stop-" + service, "before": before, "after": after})
finally:
ops.restore()
before = [ops.state(s) for s in ops.SERVICES]
results.append(exercise(c, "rolling-update", ops.rolling_update))
after = [ops.state(s) for s in ops.SERVICES]
check(all(a["container"] != b["container"] for a, b in zip(before, after)),
"both API containers must be replaced")
processes.append({"action": "rolling-update", "before": before, "after": after})
(dest / "grade4.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(dest / "processes.json").write_text(json.dumps(processes, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
suffix = ".cmd" if os.name == "nt" else ".sh"
try:
checked("tests/resilience.py", "failover.txt", "--mode", "failover",
"--action-script", str(ROOT / ("scripts/stop-one" + suffix)), *http_args)
finally:
ops.restore()
checked("tests/resilience.py", "persistence.txt", "--mode", "persistence",
"--action-script", str(ROOT / ("scripts/restart" + suffix)), *http_args)
print("PASS: both replica failures, rolling replacement and confirmed data")
if __name__ == "__main__":
run(main)