#!/usr/bin/env python3 """Stops each API separately and replaces both under a small closed-loop load.""" from concurrent.futures import ThreadPoolExecutor import json import math import os from pathlib import Path import sys import subprocess import threading import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, arguments, check, run from resilience import history import ops def exercise(c, name, action): alice, bob, eve, chat = c.bootstrap() initial_key = uuid.uuid4().hex initial = c.message(alice, chat, "До воздействия", key=initial_key)["json"] observed = {c.call("GET", "/api/v1/users/me", actor=alice)["headers"]["X-Instance-Id"] for _ in range(20)} check(len(observed) >= 2, "need two actual instances before " + name) stop = threading.Event() barrier = threading.Barrier(5) def worker(number): local = Client(c.base, timeout=2) writes, cycles, errors = [], [], 0 barrier.wait() key = uuid.uuid4().hex body = "worker-%d-%s" % (number, key) while not stop.is_set(): started = time.monotonic() try: response = local.message(alice, chat, body, key=key) writes.append(response["json"]) # Read with the original session while another backend may serve it. local.call("GET", "/api/v1/chats/" + chat["id"], actor=bob, schema="Chat") finished = time.monotonic() cycles.append((finished, finished - started, response["headers"]["X-Instance-Id"])) key = uuid.uuid4().hex body = "worker-%d-%s" % (number, key) except Failure: errors += 1 # An uncertain write keeps its original key and body. time.sleep(0.05) return writes, cycles, errors started = time.monotonic() with ThreadPoolExecutor(max_workers=4) as pool: futures = [pool.submit(worker, n) for n in range(4)] barrier.wait() try: time.sleep(1) action_started = time.monotonic() phases = action() action_finished = time.monotonic() time.sleep(7) finally: stop.set() results = [f.result() for f in futures] elapsed = time.monotonic() - started writes = [msg for messages, _, _ in results for msg in messages] cycles = sorted(item for _, values, _ in results for item in values) errors = sum(n for _, _, n in results) # An in-flight POST can finish before the stop and its paired GET after it. # Only cycles started after the action count as traffic to a surviving API. recovered = [t for t, duration, _ in cycles if t - duration >= action_finished] check(bool(recovered), "no reads/writes after " + name) recovery = recovered[0] - action_started check(recovery <= 10 if name.startswith("stop-") else True, "failover exceeded 10 seconds") times = [started] + [t for t, _, _ in cycles] + [time.monotonic()] max_gap = max(b - a for a, b in zip(times, times[1:])) check(max_gap <= 10, "no successful read/write cycle for more than 10 seconds") during = [t for t, _, _ in cycles if action_started <= t <= action_finished] if name == "rolling-update": check(bool(during), "no successful traffic during rolling replacement") for phase in phases or []: check(any(phase["started"] <= t <= phase["finished"] for t in during), "no successful traffic while replacing " + phase["service"]) stored = history(c, bob, chat) ids = {m["id"] for m in stored} check(len(ids) == len(stored), "duplicate message IDs") check(len({m["text"] for m in stored}) == len(stored), "retry created duplicate text") by_id = {m["id"]: m for m in stored} check(all(by_id.get(m["id"]) == m for m in [initial] + writes), "confirmed data lost") check(c.message(alice, chat, initial["text"], key=initial_key)["json"] == initial, "pre-action idempotency result lost") latency = sorted(d * 1000 for _, d, _ in cycles) def percentile(p): return round(latency[max(0, math.ceil(p * len(latency)) - 1)], 3) result = {"scenario": name, "concurrency": 4, "elapsed_seconds": round(elapsed, 3), "successful_read_write_cycles": len(cycles), "failed_attempts": errors, "successful_http_requests": len(writes) + len(cycles), "successful_rps": round((len(writes) + len(cycles)) / elapsed, 3), "successful_cycles_per_second": round(len(cycles) / elapsed, 3), "successful_cycles_during_action": len(during), "max_success_gap_seconds": round(max_gap, 3), "first_success_after_action_seconds": round(recovery, 3), "latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)}, "instances_before": sorted(observed), "instances_after": sorted({i for t, duration, i in cycles if t - duration >= action_finished}), "confirmed_messages_preserved": len({m["id"] for m in [initial] + writes}), "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration."} print(json.dumps(result, ensure_ascii=False, indent=2), flush=True) return result def main(): args = arguments("Lab3 grade4 Compose experiments") c = Client(args.base_url, args.ca_file) ops.BASE_URL = c.base dest = ROOT / "evidence" dest.mkdir(exist_ok=True) http_args = ["--base-url", c.base] if args.ca_file: http_args += ["--ca-file", args.ca_file] def checked(script, filename, *extra): result = subprocess.run([sys.executable, str(ROOT / script), *extra], cwd=ROOT, text=True, encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONIOENCODING="utf-8")) (dest / filename).write_text(result.stdout, encoding="utf-8") print(result.stdout, end="", flush=True) check(result.returncode == 0, script + " failed; see evidence/" + filename) ops.restore() checked("scripts/check_contract.py", "check.txt") checked("tests/smoke.py", "smoke.txt", *http_args) checked("student-tests/run.py", "student-tests.txt", *http_args) results = [] processes = [] for service in ops.SERVICES: ops.restore() before = ops.state(service) try: result = exercise(c, "stop-" + service, lambda: ops.stop_one(service)) after = ops.state(service) check(after["status"] == "exited", "stopped API restarted unexpectedly") check(all(i.split(":", 1)[0] != before["hostname"] for i in result["instances_after"]), "stopped replica still handles requests") results.append(result) processes.append({"action": "stop-" + service, "before": before, "after": after}) finally: ops.restore() before = [ops.state(s) for s in ops.SERVICES] results.append(exercise(c, "rolling-update", ops.rolling_update)) after = [ops.state(s) for s in ops.SERVICES] check(all(a["container"] != b["container"] for a, b in zip(before, after)), "both API containers must be replaced") processes.append({"action": "rolling-update", "before": before, "after": after}) (dest / "grade4.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (dest / "processes.json").write_text(json.dumps(processes, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") suffix = ".cmd" if os.name == "nt" else ".sh" try: checked("tests/resilience.py", "failover.txt", "--mode", "failover", "--action-script", str(ROOT / ("scripts/stop-one" + suffix)), *http_args) finally: ops.restore() checked("tests/resilience.py", "persistence.txt", "--mode", "persistence", "--action-script", str(ROOT / ("scripts/restart" + suffix)), *http_args) print("PASS: both replica failures, rolling replacement and confirmed data") if __name__ == "__main__": run(main)