#!/usr/bin/env python3 """Явные эксперименты lab3: останавливает и восстанавливает API, заменяет контейнеры.""" import argparse import http.client from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import json import os from pathlib import Path import platform import subprocess import sys import threading import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check from resilience import history from replicas import ( COMPOSE, SERVICES, compose, inventory, probe, replace, undrain, wait_balanced, wait_ready, ) def save_json(path, value): path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n") def record_command(out, name, command, env=None): """Записать реальный вывод команды; не запрашивать полный inspect или env.""" started = time.monotonic() result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, env=env) (out / (name + ".txt")).write_text( "$ " + " ".join(command) + "\n" + result.stdout + result.stderr + f"\nexit_code={result.returncode}; elapsed_seconds={time.monotonic() - started:.3f}\n" ) check(result.returncode == 0, name + " failed; inspect evidence file") def observe(c, actor): seen = set() for _ in range(50): try: resp = c.call("GET", "/api/v1/users/me", actor=actor, schema="User") instance_id = resp["headers"].get("X-Instance-Id") if instance_id: seen.add(instance_id) except Exception: pass if len(seen) >= 2: break time.sleep(0.1) check(len(seen) >= 2, "one session must be accepted by at least two API processes") return sorted(seen) def concurrent_replay(base): """Одновременный один ключ через LB; один id, одна строка истории, обе реплики.""" c = Client(base) alice, bob, eve, chat = c.bootstrap() key = uuid.uuid4().hex barrier = threading.Barrier(12) def worker(_): local = Client(base) barrier.wait() return local.message(alice, chat, "одновременный повтор", key=key) with ThreadPoolExecutor(max_workers=12) as pool: responses = list(pool.map(worker, range(12))) ids = {r["json"]["id"] for r in responses} instances = {r["headers"]["X-Instance-Id"] for r in responses} check(len(ids) == 1, "concurrent replay created different messages") check(len(instances) >= 2, "replay did not cross two API processes") rows = history(c, bob, chat) check( len(rows) == 1 and rows[0] == responses[0]["json"], "history must contain one exact response", ) c.call( "POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, body={"text": "другое тело"}, headers={"Idempotency-Key": key}, ) replay = c.message(alice, chat, "одновременный повтор", key=key) check(replay["json"] == responses[0]["json"], "409 changed original result") return { "parallel_requests": 12, "unique_message_ids": len(ids), "history_rows": len(rows), "instances": sorted(instances), "conflict_status": 409, } def experiment(base, service, mode, duration): """Closed-loop чтение/запись во время stop или поочерёдной замены.""" c = Client(base, timeout=2) alice, bob, eve, chat = c.bootstrap() initial = c.message(alice, chat, "подтверждено до действия")["json"] seen = observe(c, alice) before = inventory() target = probe(service)["instance_id"] events, confirmed = [], [initial] lock = threading.Lock() halt = threading.Event() started = time.monotonic() def worker(number): local = Client(base, timeout=2) key, text = uuid.uuid4().hex, "" while not halt.is_set(): if not text: text = "load-" + key for operation in ("write", "read"): t0 = time.monotonic() try: response = ( local.message(alice, chat, text, key=key) if operation == "write" else local.call( "GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=1", actor=bob, schema="MessagePage", ) ) event = { "start": t0, "end": time.monotonic(), "kind": operation, "ok": True, "instance": response["headers"]["X-Instance-Id"], } with lock: if operation == "write": confirmed.append(response["json"]) if operation == "write": key, text = uuid.uuid4().hex, "" except (Failure, OSError, http.client.HTTPException): event = { "start": t0, "end": time.monotonic(), "kind": operation, "ok": False, } with lock: events.append(event) time.sleep(0.02) with ThreadPoolExecutor(max_workers=2) as pool: futures = [pool.submit(worker, n) for n in range(2)] time.sleep(1) action_started = time.monotonic() try: if mode == "stop": env = {**os.environ, "TARGET_INSTANCE_ID": target} # Вызываем replicas.py stop-target напрямую через текущий интерпретатор Python result = subprocess.run( [sys.executable, str(ROOT / "scripts/replicas.py"), "stop-target"], cwd=ROOT, env=env, text=True, capture_output=True, timeout=10, ) check(result.returncode == 0, "stop hook failed") action_detail = result.stdout else: action_detail = replace(service) action_finished = time.monotonic() after_action = inventory() if mode == "stop": stopped = next(row for row in after_action if row["service"] == service) check( stopped["state"] == "exited" and stopped["pid"] == 0, "target process is still running", ) deadline = max(action_started + duration, action_finished + 5) while time.monotonic() < deadline: time.sleep(0.1) finally: halt.set() for future in futures: future.result() finished = time.monotonic() rows = history(c, bob, chat) by_id = {row["id"]: row for row in rows} check( all(by_id.get(message["id"]) == message for message in confirmed), "confirmed data was lost", ) texts = [row["text"] for row in rows] check(len(texts) == len(set(texts)), "uncertain retry created duplicate text") tail = [e for e in events if e["start"] >= action_finished] check(any(e["ok"] and e["kind"] == "write" for e in tail), "no writes after action") check(any(e["ok"] and e["kind"] == "read" for e in tail), "no reads after action") if mode == "stop": check( all(e.get("instance") != target for e in tail if e["ok"]), "stopped instance still responds", ) for kind in ("write", "read"): recovered = min(e["end"] for e in tail if e["ok"] and e["kind"] == kind) check(recovered - action_started <= 10, kind + " recovery exceeded 10s") def metrics(items, elapsed): good = [e for e in items if e["ok"]] failed = len(items) - len(good) latencies = sorted((e["end"] - e["start"]) * 1000 for e in good) def percentile(fraction): return ( round(latencies[max(0, int(len(latencies) * fraction + 0.999) - 1)], 3) if latencies else None ) return { "elapsed_seconds": round(elapsed, 3), "attempts": len(items), "successes": len(good), "errors": failed, "error_ratio": round(failed / len(items), 6) if items else None, "successful_rps": round(len(good) / elapsed, 3), "successful_latency_ms": { "p50": percentile(0.5), "p95": percentile(0.95), "p99": percentile(0.99), }, } result = { "mode": mode, "service": service, "target_instance": target, "observed_with_one_session": seen, "before": before, "after_action": after_action, "action": action_detail, "action_seconds": round(action_finished - action_started, 3), "concurrency": 2, "total": metrics(events, finished - started), "during_and_after_action": metrics( [e for e in events if e["start"] >= action_started], finished - action_started, ), "confirmed_responses": len(confirmed), "history_messages": len(rows), "duplicates": 0, "existing_session_works": True, } for kind in ("write", "read"): after = [e for e in tail if e["ok"] and e["kind"] == kind] result["first_" + kind + "_after_action_seconds"] = round( min(e["end"] for e in after) - action_started, 3 ) result[kind] = metrics( [e for e in events if e["kind"] == kind], finished - started ) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--output", default="evidence/lab3") parser.add_argument("--duration", type=float, default=20) args = parser.parse_args() check(15 <= args.duration <= 120, "duration must be 15..120 seconds") out = ROOT / args.output out.mkdir(parents=True, exist_ok=True) save_json(out / "result.json", {"status": "RUNNING"}) try: os.environ["BASE_URL"] = args.base_url save_json( out / "00_environment.json", { "at_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(), "cpu_count": os.cpu_count(), "python": platform.python_version(), "duration": args.duration, "workload": "2 closed-loop clients, POST+GET; uncertain POST retried with same key/body", }, ) record_command(out, "01_docker", ["docker", "version"]) record_command(out, "02_compose_config", COMPOSE + ["config", "--quiet"]) wait_balanced() save_json(out / "03_processes.json", inventory()) record_command(out, "03_limits", COMPOSE + ["ps", "-q"]) ids = compose("ps", "-q", capture=True).splitlines() record_command( out, "03_container_limits", [ "docker", "inspect", "--format", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} ports={{json .HostConfig.PortBindings}}", *ids, ], ) record_command( out, "04_smoke", [sys.executable, "tests/smoke.py", "--base-url", args.base_url], ) save_json( out / "05_concurrent_idempotency.json", concurrent_replay(args.base_url) ) try: for service in SERVICES: wait_balanced() report = experiment(args.base_url, service, "stop", args.duration) save_json(out / ("06_failover_" + service + ".json"), report) compose("up", "-d", "--no-deps", service) wait_balanced() record_command( out, "07_public_failover", [ sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", "scripts/stop-one.sh", "--base-url", args.base_url, ], ) for service in SERVICES: compose("up", "-d", "--no-deps", service) wait_balanced() for service in SERVICES: save_json( out / ("08_rolling_" + service + ".json"), experiment(args.base_url, service, "replace", args.duration), ) record_command( out, "09_persistence", [ sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", "scripts/restart.sh", "--base-url", args.base_url, ], ) wait_balanced() save_json(out / "10_processes_after.json", inventory()) record_command( out, "11_nginx", COMPOSE + ["exec", "-T", "nginx", "nginx", "-T", "-c", "/opt/lb/nginx.conf"], ) record_command( out, "12_student_tests", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "tests"], ) save_json( out / "result.json", {"status": "PASS", "at_utc": datetime.now(timezone.utc).isoformat()}, ) except Exception: save_json( out / "result.json", { "status": "FAIL", "note": "Inspect completed evidence files; failed runs are not evidence of passing.", }, ) raise finally: for service in SERVICES: compose("up", "-d", "--no-deps", service) undrain(service) print("PASS: evidence saved to " + str(out.relative_to(ROOT))) except BaseException: save_json( out / "result.json", { "status": "FAIL", "note": "Run did not finish successfully; inspect completed outputs.", }, ) raise if __name__ == "__main__": main()