"""Проверки уровня 4 в отдельном Compose project с синтетическими данными.""" from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import json import os import platform import socket import subprocess import sys import threading import time from urllib.error import HTTPError from urllib.request import urlopen from uuid import uuid4 from helpers import Client, ROOT from client import Failure from resilience import history from attachment_checks import run_checks from postgres_pause import check_postgres_pause def docker(*args): return subprocess.check_output(["docker", *args], text=True, timeout=180).strip() def main(): with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] project = "lab4-check-" + uuid4().hex[:10] base = f"http://127.0.0.1:{port}" env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base) result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project, "base_url": base, "host": platform.system() + " " + platform.machine(), "docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"), "commit": docker("compose", "version", "--short"), "profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}} result["compose_version"] = result.pop("commit") result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() result["working_tree"] = True def compose(*args): return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() def script(name): subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240) def inspect_all(): ids = compose("ps", "-a", "-q").split() return json.loads(docker("inspect", *ids)) def apis(): return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"] def metadata(info): return {"id": info["Id"], "instance": info["Config"]["Hostname"], "service": info["Config"]["Labels"]["com.docker.compose.service"], "image": info["Image"], "started_at": info["State"]["StartedAt"], "running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"], "memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"], "published_ports": info["HostConfig"]["PortBindings"], "mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]} def sql(query): return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query) def check_shutdown(): info = apis()[0] # Внутреннее подключение через docker exec не публикует порт API на хост. pending_code = """import socket,time s=socket.create_connection(('127.0.0.1',8080),timeout=2) s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n') assert b'100 Continue' in s.recv(1024) s.sendall(b'{"username":') print('pending',flush=True) time.sleep(30) """ pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: assert pending.stdout.readline().strip() == "pending" started = time.monotonic() docker("kill", "--signal=TERM", info["Id"]) first_refusal = None while True: state = json.loads(docker("inspect", info["Id"]))[0]["State"] if not state["Running"]: break assert time.monotonic() - started < 8 probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c", "import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"], capture_output=True, timeout=3) if probe.returncode and first_refusal is None: first_refusal = time.monotonic() - started time.sleep(0.1) elapsed = time.monotonic() - started assert first_refusal is not None and first_refusal < elapsed < 8 assert state["ExitCode"] == 0 and not state["OOMKilled"] logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) assert "Application shutdown complete" in logs.stdout + logs.stderr result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True, "shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3), "exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]} print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True) finally: pending.communicate(timeout=10) compose("start", "api") script("wait-ready.sh") def load_action(label, action, target=None): client = Client(base, timeout=2) alice, bob, _, chat = client.bootstrap() original = client.message(alice, chat, "До воздействия")["json"] stop = threading.Event() lock = threading.Lock() successes, errors, confirmed = [], [], {} started = time.monotonic() def worker(index): c = Client(base, timeout=2) key = uuid4().hex while not stop.is_set(): for method in ("POST", "GET"): try: if method == "POST": response = c.message(alice, chat, "load-" + key, key=key) with lock: confirmed[response["json"]["id"]] = response["json"] key = uuid4().hex else: response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage") with lock: successes.append((time.monotonic() - started, method, response["headers"]["X-Instance-Id"], response["elapsed"])) except Failure as exc: with lock: errors.append((time.monotonic() - started, method, str(exc))) stop.wait(0.05) with ThreadPoolExecutor(max_workers=4) as pool: futures = [pool.submit(worker, i) for i in range(4)] time.sleep(1) action_start = time.monotonic() - started try: action_info = action() action_end = time.monotonic() - started # Охватывает обнаружение отказа и устойчивую работу после него. time.sleep(max(3, 15 - (action_end - action_start))) finally: stop.set() for future in futures: future.result() elapsed = time.monotonic() - started items = history(client, bob, chat) by_id = {m["id"]: m for m in items} assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль" assert by_id.get(original["id"]) == original assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения" after = [s for s in successes if s[0] >= action_end] assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия" recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3) for method in ("POST", "GET")} assert max(recovery.values()) <= 10, recovery max_gap = {} for method in ("POST", "GET"): moments = sorted(s[0] for s in successes if s[1] == method) max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3) assert max(max_gap.values()) <= 10, max_gap if target: assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает" buckets = [] for second in range(int(elapsed) + 1): buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes), "errors": sum(second <= e[0] < second + 1 for e in errors)}) latencies = sorted(s[3] * 1000 for s in successes) summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes), "successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2), "first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap, "action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3), "p50_ms": round(latencies[len(latencies) // 2], 3), "p95_ms": round(latencies[int(len(latencies) * .95)], 3), "confirmed_data_preserved": True, "duplicate_messages": 0, "instances": sorted({s[2] for s in successes}), "per_second": buckets, "error_samples": errors[:5], "action": action_info} print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True) return summary print("Command: ./scripts/check-docker.sh; project:", project, flush=True) print("Docker:", result["docker"], flush=True) try: script("start.sh") initial = inspect_all() result["containers"] = [metadata(info) for info in initial] assert len(apis()) == 2 for info in initial: assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0 if info["Config"]["Labels"]["com.docker.compose.service"] != "lb": assert not info["HostConfig"]["PortBindings"] for info in apis(): assert docker("exec", info["Id"], "id", "-u") == "10001" assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"]) print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True) subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240) result["api_tests"] = "public smoke + student unittest suite passed" subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30) assert sql("SELECT count(*) FROM schema_migrations") == "3" count = sql("SELECT count(*) FROM messages") for _ in range(2): compose("run", "--rm", "migrate") assert sql("SELECT count(*) FROM schema_migrations") == "3" assert sql("SELECT count(*) FROM messages") == count print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True) result["attachments"] = run_checks(base, env, compose, apis, script, sql) replay_client = Client(base) replay_actor, _, _, replay_chat = replay_client.bootstrap() replay_key = uuid4().hex saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"], cwd=ROOT, env=env, check=True, timeout=180) after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} assert before.keys() == after.keys() assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before) assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True, "idempotency_replay_after_restart": True} print("PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved", flush=True) check_shutdown() dependency_client = Client(base) actor = dependency_client.user("deps") result["dependency_readiness"] = [] for dependency in ("valkey", "postgres", "s3"): try: compose("stop", "--timeout", "8", dependency) probe_id = apis()[0]["Id"] readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10) assert readiness.returncode == 1 live = docker("exec", probe_id, "python", "-c", "from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)") assert live == "200" deadline = time.monotonic() + 8 while True: try: urlopen(base + "/health/ready", timeout=2).close() except HTTPError as error: if error.code == 503: break raise except OSError: # Уже отправленный запрос может истечь до исключения backend. pass assert time.monotonic() < deadline time.sleep(0.1) result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503}) print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True) finally: compose("start", dependency) script("wait-ready.sh") dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User") result["postgres_pause"] = check_postgres_pause(compose, apis, base) subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"], cwd=ROOT, env=env, check=True, timeout=90) assert sum(i["State"]["Running"] for i in apis()) == 1 result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]} compose("start", "api") script("wait-ready.sh") result["failovers"] = [] for info in apis(): target = info["Config"]["Hostname"] def stop_target(): subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10) stopped = json.loads(docker("inspect", info["Id"]))[0] assert not stopped["State"]["Running"] assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0 # Uvicorn пишет в stderr; получаем оба потока без вывода запросов. logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr return metadata(stopped) result["failovers"].append(load_action("stop " + target, stop_target, target)) compose("start", "api") script("wait-ready.sh") before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} def replace(): script("rolling-update.sh") after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} old_api = {i["id"] for i in before.values() if i["service"] == "api"} new_api = {i["id"] for i in after.values() if i["service"] == "api"} assert len(new_api) == 2 and not old_api & new_api for cid, info in before.items(): if info["service"] != "api": assert after[cid]["started_at"] == info["started_at"] return {"before": list(before.values()), "after": list(after.values())} result["rolling_update"] = load_action("sequential API replacement", replace) result["passed"] = True print("PASS: all lab4 grade-4 checks completed", flush=True) finally: output = ROOT / "evidence" / "lab4-checks.json" output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") compose("down", "--volumes", "--timeout", "8") print("Removed only temporary test containers and volumes:", project, flush=True) if __name__ == "__main__": main()