import argparse from datetime import datetime, timezone import hashlib import json import os from pathlib import Path import platform import secrets import shutil import socket import subprocess import sys import time from urllib.error import URLError from urllib.request import ProxyHandler, build_opener import venv ROOT = Path(__file__).resolve().parents[1] LOG = [] LOCAL_HTTP = build_opener(ProxyHandler({})) def settings(): path = ROOT / ".env" result = {} if path.exists(): for line in path.read_text(encoding="utf-8-sig").splitlines(): key, separator, value = line.partition("=") if separator and not key.strip().startswith("#"): result[key.strip()] = value.strip() return result def environment(args): result = os.environ.copy() result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1", COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port), SESSION_TTL_SECONDS=str(args.ttl), LAB3_PROJECT=args.project, LAB3_PORT=str(args.port), LAB3_TTL=str(args.ttl)) for key in ("NO_PROXY", "no_proxy"): result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"])) return result def say(value): line = str(value).replace(str(ROOT), ".") print(line, flush=True) LOG.append(line) def run(args, command, quiet=False): command = [str(item) for item in command] if not quiet: say("$ " + subprocess.list2cmdline(command)) with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process: lines = [] for line in process.stdout: lines.append(line) if not quiet: say(line.rstrip()) code = process.wait() if code: raise RuntimeError(f"{command[0]} exited with code {code}") return "".join(lines) def compose(args, *command, quiet=False): if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"): raise RuntimeError("This runner only manages Kuznetsov lab3 projects") return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet) def interpreter(): return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") def evidence(name, value): path = ROOT / "evidence" / name path.parent.mkdir(exist_ok=True) content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n") def setup(args): if not (ROOT / ".env").exists(): shutil.copyfile(ROOT / ".env.example", ROOT / ".env") folder = ROOT / "secrets" folder.mkdir(exist_ok=True) password_file = folder / "postgres_password" if not password_file.exists(): password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii") password = password_file.read_text(encoding="ascii").strip() values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger", "cursor_key": secrets.token_hex(32)} for name, value in values.items(): path = folder / name if not path.exists(): path.write_text(value + "\n", encoding="ascii") if os.name != "nt": path.chmod(0o644) if os.name != "nt": folder.chmod(0o700) password_file.chmod(0o644) if not interpreter().exists(): venv.EnvBuilder(with_pip=True).create(ROOT / ".venv") lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"] fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest() marker = ROOT / ".venv/.lab-dependencies" if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint: run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]]) marker.write_text(fingerprint, encoding="ascii") say("Environment ready; existing settings and secrets preserved") def base_url(args): return f"http://127.0.0.1:{args.port}" def wait_ready(args, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response: if response.status == 200 and json.load(response) == {"status": "ok"}: return except (URLError, OSError, ValueError): pass time.sleep(0.3) raise RuntimeError("API readiness timeout") def build(args): compose(args, "config", "--quiet") compose(args, "build", "api1") compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg") def start(args): current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip() if not current: with socket.socket() as probe: try: probe.bind(("127.0.0.1", args.port)) except OSError: raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None compose(args, "up", "-d", "--no-build", "db", "migrate", "api1", "api2", "lb") wait_ready(args) wait_replicas(args) say("Two API replicas ready through LB: " + base_url(args)) def states(args): result = {} for service in ("api1", "api2", "db", "lb"): container = compose(args, "ps", "--all", "--quiet", service, quiet=True).strip() if not container: raise RuntimeError("Missing container: " + service) data = json.loads(run(args, ["docker", "inspect", container], quiet=True))[0] labels = data["Config"].get("Labels", {}) if labels.get("com.docker.compose.project") != args.project or labels.get("com.docker.compose.service") != service: raise RuntimeError("Unexpected container ownership") result[service] = { "id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"], "running": data["State"]["Running"], "user": data["Config"]["User"], "volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"], } return result def restart(args): before = states(args) if not all(item["running"] for item in before.values()): raise RuntimeError("All runtime containers must be running before restart") compose(args, "restart", "--timeout", "10", "db", "api1", "api2", "lb") after = states(args) for service in before: if before[service]["started_at"] == after[service]["started_at"]: raise RuntimeError("Container was not restarted: " + service) if before[service]["volumes"] != after[service]["volumes"]: raise RuntimeError("Volume changed during restart") evidence("restart.json", {"project": args.project, "before": before, "after": after}) say("PASS: both APIs, PostgreSQL and LB restarted, volume preserved") def contract_tests(args): run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"]) run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"]) compose(args, "config", "--quiet") def api_tests(args): wait_ready(args) run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)]) os.environ["BASE_URL"] = base_url(args) run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"]) def client(args): sys.path.insert(0, str(ROOT / "tests")) from client import Client for key in ("NO_PROXY", "no_proxy"): os.environ[key] = environment(args)[key] return Client(base_url(args)) def database_stats(args): sql = """SELECT json_build_object( 'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats), 'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages), 'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations), 'idempotency_keys',(SELECT count(*) FROM idempotency_keys), 'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text, ',' ORDER BY token_hash),'')) FROM sessions), 'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users), 'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))""" result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql, quiet=True) return json.loads(result) def restart_check(args): from urllib.parse import urlencode c = client(args) alice, bob, eve, chat = c.bootstrap() path = "/api/v1/chats/" + chat["id"] + "/messages" replay_key = secrets.token_hex(16) original = c.message(alice, chat, "first", key=replay_key)["json"] last = c.message(bob, chat, "second")["json"] page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"] revoked = c.user("revoked") c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked) hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", base_url(args)]) continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}), actor=alice, schema="MessagePage")["json"] if continued != {"items": [last], "next_cursor": None}: raise RuntimeError("Old cursor did not survive restart") c.call("GET", "/api/v1/users/me", 401, actor=revoked) c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve) before = database_stats(args) compose(args, "run", "--rm", "--no-deps", "migrate") compose(args, "run", "--rm", "--no-deps", "migrate") after = database_stats(args) if before != after or after["migrations"] != 2: raise RuntimeError("Migration replay changed existing data") if not after["passwords_are_argon2id"] or not after["session_hashes_only"]: raise RuntimeError("Unexpected credential representation in DB") repeated = c.message(alice, chat, "first", key=replay_key)["json"] if repeated != original: raise RuntimeError("Idempotency result was not preserved") result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS", "logout_after_restart": "PASS", "migration_replay": "PASS", "database": after} evidence("persistence.json", result) say("PASS: old session/cursor, durable logout and repeatable migrations") return result def ttl_check(args): test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2, test_port=args.test_port, action="start") try: start(test) c = client(test) actor = c.user("expiry") c.call("GET", "/api/v1/users/me", actor=actor, schema="User") before = database_stats(test)["expiry_fingerprint"] compose(test, "restart", "--timeout", "3", "api1", "api2") wait_ready(test) after = database_stats(test)["expiry_fingerprint"] if before != after: raise RuntimeError("Session expiry changed during restart") time.sleep(2.2) c.call("GET", "/api/v1/users/me", 401, actor=actor) registration = actor["registration"] import base64 basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode() c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session") say("PASS: TTL=2, expired token rejected after API restart; fresh login works") result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200, "after_expiry_and_restart": 401, "new_login": 201} evidence("expiry.json", result) return result finally: compose(test, "down", "--timeout", "10") def dependency_check(args): c = client(args) actor = c.user("dbcheck") try: compose(args, "stop", "--timeout", "10", "db") c.call("GET", "/health/live", schema="Health") outage_response(args, "/health/ready") outage_response(args, "/api/v1/users/me", actor["token"]) finally: compose(args, "start", "db") wait_ready(args) c.call("GET", "/api/v1/users/me", actor=actor, schema="User") say("PASS: DB outage -> live=200, ready/business=503; session works after recovery") return {"live_without_db": 200, "ready_without_db": 503, "business_without_db": 503, "recovered": 200} def verify(args): summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(), "source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(), "worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()), "project": args.project, "status": "FAIL"} try: setup(args) contract_tests(args) build(args) start(args) api_tests(args) summary["persistence"] = restart_check(args) summary["failover"] = failover_check(args) summary["expiry"] = ttl_check(args) summary["dependency"] = dependency_check(args) run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)]) summary["containers"] = states(args) summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip() summary["resources"] = json.loads(run( args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True)) summary["status"] = "PASS" say("PASS: full lab3 verification") finally: evidence("verification.txt", "\n".join(LOG)) evidence("summary.json", summary) def instance_map(args): current = states(args) result = {} for service in ("api1", "api2"): item = current[service] if not item["running"]: continue code = ( "import urllib.request; " "o=urllib.request.build_opener(urllib.request.ProxyHandler({})); " "r=o.open('http://127.0.0.1:8080/health/live',timeout=1); " "print(r.headers['X-Instance-Id'])" ) alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip() import re if not re.fullmatch(r"[0-9a-f]{32}", alias): raise RuntimeError("Invalid API instance alias") result[service] = {"instance_id": alias, "container_id": item["id"]} return result def wait_replicas(args): deadline = time.monotonic() + 90 while time.monotonic() < deadline: try: mapping = instance_map(args) if len(mapping) == 2: c = client(args) seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"] for _ in range(8)} if seen == {item["instance_id"] for item in mapping.values()}: return mapping except (RuntimeError, OSError, AssertionError): pass time.sleep(0.3) raise RuntimeError("Two healthy API replicas were not observed through LB") def restore(args): compose(args, "start", "api1", "api2") wait_ready(args) return wait_replicas(args) def stop_one(args): target = os.environ.get("TARGET_INSTANCE_ID", "") before = instance_map(args) matches = [(name, item) for name, item in before.items() if item["instance_id"] == target] if len(before) != 2 or len(matches) != 1: raise RuntimeError("Target must match exactly one of two running API replicas") service, item = matches[0] started = time.monotonic() run(args, ["docker", "stop", "--timeout", "3", item["container_id"]]) after = states(args) if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]: raise RuntimeError("Unexpected API state after stop") elapsed = time.monotonic() - started if elapsed >= 10: raise RuntimeError("Stopping the selected API exceeded 10 seconds") evidence("stop-one.json", {"service": service, "target_instance": target, "before": before, "after": after, "stop_seconds": round(elapsed, 3)}) say("Stopped only " + service) def failover_check(args): wait_replicas(args) hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") try: output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", base_url(args)]) first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8")) first["metrics"] = next(json.loads(line) for line in output.splitlines() if line.startswith("{") and "first_recovery_seconds" in line) if first["after"][first["service"]]["running"]: raise RuntimeError("Public hook did not leave the target stopped") finally: restore(args) other_service = "api2" if first["service"] == "api1" else "api1" mapping = instance_map(args) target = mapping[other_service]["instance_id"] c = client(args) c.timeout = 2 alice, bob, eve, chat = c.bootstrap() for _ in range(10): replay_key = secrets.token_hex(16) replay_text = "before-failure-" + replay_key response = c.message(alice, chat, replay_text, key=replay_key) if response["headers"]["X-Instance-Id"] == target: original = response["json"] break else: raise RuntimeError("Second API was not reached through LB") env = environment(args) env["TARGET_INSTANCE_ID"] = target started = time.monotonic() process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) from client import Failure from resilience import history confirmations, after_instances = [], [] failures = 0 recovered = None key = secrets.token_hex(16) text = "during-failure-" + key try: while time.monotonic() - started < 15: code = process.poll() if code not in (None, 0): raise RuntimeError("Second stop hook failed") if code is None and time.monotonic() - started >= 10: raise RuntimeError("Second stop hook exceeded 10 seconds") try: response = c.message(alice, chat, text, key=key) confirmations.append(response["json"]) if code == 0: if recovered is None: recovered = time.monotonic() - started after_instances.append(response["headers"]["X-Instance-Id"]) key = secrets.token_hex(16) text = "during-failure-" + key except Failure: failures += 1 time.sleep(0.1) if process.poll() != 0 or recovered is None or recovered > 10: raise RuntimeError("Second API failure did not recover within 10 seconds") if len(after_instances) < 5 or target in after_instances[-5:]: raise RuntimeError("Stopped API still serves requests") replayed = c.message(alice, chat, replay_text, key=replay_key) if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target: raise RuntimeError("Message replay through the survivor changed the result") messages = history(c, bob, chat) ids = [item["id"] for item in messages] texts = [item["text"] for item in messages] if len(ids) != len(set(ids)) or len(texts) != len(set(texts)): raise RuntimeError("Duplicates appeared during failover") if any(item not in messages for item in [original, *confirmations]): raise RuntimeError("A confirmed message was lost") second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8")) if second["service"] != other_service: raise RuntimeError("Wrong second API was stopped") second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures, "first_recovery_seconds": round(recovered, 3), "replay_on_survivor": "PASS"} result = {"public": first, "second_replica": second, "status": "PASS"} evidence("failover.json", result) say("PASS: both physical API replicas stopped separately; no lost messages or duplicates") return result finally: if process.poll() is None: process.terminate() process.wait(timeout=5) restore(args) def outage_response(args, path, token=None): from urllib.error import HTTPError from urllib.request import Request from client import shape headers = {"Authorization": "Bearer " + token} if token else {} try: response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10) except HTTPError as error: response = error with response: payload = json.load(response) if response.status != 503 or not response.headers.get("X-Request-Id"): raise RuntimeError("Unexpected dependency failure response") shape(payload, "Error") if payload["error"]["request_id"] != response.headers["X-Request-Id"]: raise RuntimeError("LB error request ID mismatch") def main(): saved = settings() parser = argparse.ArgumentParser() parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one", "restart-check", "failover-check", "restore", "stop", "verify"]) parser.add_argument("--project", default=os.environ.get("LAB3_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "kuznetsov-lab3"))) parser.add_argument("--port", type=int, default=int(os.environ.get("LAB3_PORT", saved.get("HTTP_PORT", "18103")))) parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB3_TTL", saved.get("SESSION_TTL_SECONDS", "3600")))) parser.add_argument("--test-port", type=int, default=18113) args = parser.parse_args() if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"): parser.error("Use a dedicated Kuznetsov lab3 project") if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65535 or not 1 <= args.ttl <= 86400: parser.error("Invalid port or session TTL") if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check"): setup(args) actions = {"setup": lambda item: None, "build": build, "start": start, "test": lambda item: (contract_tests(item), api_tests(item)), "restart": restart, "restart-check": restart_check, "verify": verify, "stop-one": stop_one, "restore": restore, "failover-check": failover_check, "stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "db")} actions[args.action](args) if __name__ == "__main__": try: main() except (RuntimeError, OSError, ValueError) as error: print("FAIL: " + str(error), file=sys.stderr) raise SystemExit(1)