"""Shared helpers for the student tests. * `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services. * `test_*` in this folder — marker `stack`: black-box HTTP against the running compose stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database checks (psql inside the postgres container), the HAProxy runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP. python -m pytest student-tests # everything (stand must be up) python -m pytest student-tests -m "not stack" # unit only """ from __future__ import annotations import base64 import contextlib import json import os import subprocess import sys import time import uuid from pathlib import Path from urllib.parse import quote import httpx2 as httpx import pytest from jsonschema import Draft202012Validator ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) sys.path.insert(0, str(ROOT / "scripts")) import lbctl # noqa: E402 SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8")) BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs def pytest_configure(config): config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)") def schema_errors(value, name: str) -> list[str]: """Validate against components/schemas/ of the course contract (JSON Schema 2020-12).""" schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]} validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER) return [e.message for e in validator.iter_errors(value)] def basic(username: str, password: str) -> str: return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() # ------------------------------------------------------------------ docker / psql def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess: return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=check, timeout=timeout, input=stdin) def psql(sql: str) -> str: """Run SQL inside the postgres container (no DB port is published) and return unaligned output.""" result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", "ON_ERROR_STOP=1", "-c", sql) return result.stdout.strip() def wait_ready(timeout: float = 90) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200: return except httpx.HTTPError: pass time.sleep(0.3) raise AssertionError("stand did not become ready") def wait_healthy(*services: str, timeout: float = 120) -> None: """Poll Docker health until EVERY container of every service is healthy again. `compose up --wait` is not used: it gives up at once on a container marked unhealthy while paused.""" deadline = time.monotonic() + timeout pending = set(services) while pending and time.monotonic() < deadline: for service in list(pending): ids = compose("ps", "-q", service, check=False).stdout.split() if ids: states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids], capture_output=True, text=True).stdout.split() if states and all(state == "healthy" for state in states): pending.discard(service) time.sleep(0.5) assert not pending, f"not healthy: {pending}" def wait_stand(replicas: int = REPLICAS) -> None: """Readiness through the LB AND every replica back in HAProxy rotation.""" wait_ready() lbctl.wait_up(replicas, timeout=90) def restart_services(*services: str) -> None: compose("restart", *services) wait_healthy(*services) wait_stand() @contextlib.contextmanager def service_down(service: str, how: str = "stop"): """Stop or pause a compose service; ALWAYS bring it back and wait for readiness.""" assert how in {"stop", "pause"} compose(how, service) try: yield finally: compose("unpause" if how == "pause" else "start", service) wait_healthy(service) wait_stand() @contextlib.contextmanager def api_with_env(**overrides: str): """Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2), then ALWAYS recreate it with the normal configuration.""" env = {**os.environ, **overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, capture_output=True, text=True, check=True, timeout=180) wait_stand() try: yield finally: clean = {k: v for k, v in os.environ.items() if k not in overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, capture_output=True, text=True, check=True, timeout=180) wait_stand() def running_replicas() -> list[dict]: return [r for r in lbctl.replicas() if r["state"] == "running"] @contextlib.contextmanager def replica_stopped(replica: dict, mode: str = "stop"): """Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again and wait until HAProxy has every replica UP.""" assert mode in {"stop", "kill"} subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]], capture_output=True, text=True, check=True, timeout=60) try: yield finally: subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60) wait_healthy("api") wait_stand() def api_env(name: str) -> str: """Current value of an environment variable inside the running api container.""" return compose("exec", "-T", "api", "printenv", name).stdout.strip() # ------------------------------------------------------------------ HTTP class Api: """HTTP helper that checks the invariants every response must satisfy.""" def __init__(self, base_url: str = BASE_URL): self.client = httpx.Client(base_url=base_url, timeout=15) def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None, lb_error_ok: bool = False): """lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is accepted without X-Instance-Id; every response of an API replica must carry it.""" h = dict(headers or {}) if token is not None: h["Authorization"] = "Bearer " + token if json_body is not None: raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8") h.setdefault("Content-Type", "application/json") response = self.client.request(method, path, content=raw, headers=h) allowed = (expected,) if isinstance(expected, int) else expected assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) request_id = response.headers.get("x-request-id") assert request_id, "every response must carry X-Request-Id" if not response.headers.get("x-instance-id"): assert lb_error_ok and response.status_code in (502, 503, 504), \ ("missing X-Instance-Id", method, path, response.status_code) if response.status_code >= 400: body = response.json() assert schema_errors(body, "Error") == [], response.text assert body["error"]["request_id"] == request_id assert "Traceback" not in response.text and "psycopg" not in response.text if response.status_code == 401: challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer" assert response.headers.get("www-authenticate", "").lower().startswith(challenge) elif schema: assert schema_errors(response.json(), schema) == [], response.text return response def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict: username = f"{label}_{uuid.uuid4().hex[:12]}"[:32] body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password} user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json() return {"id": user["id"], "user": user, "username": username, "password": password} def login(self, actor: dict) -> str: session = self.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json() assert session["user"] == actor["user"] and session["token_type"] == "Bearer" return session["token"] def user(self, label: str = "u", **kwargs) -> dict: actor = self.register(label, **kwargs) actor["token"] = self.login(actor) return actor def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict: ids = [owner["id"], *(m["id"] for m in members)] return self.call("POST", "/api/v1/chats", 201, token=owner["token"], json_body={"title": title, "member_ids": ids}, schema="Chat").json() def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None, raw: bytes | None = None, headers: dict | None = None, **kwargs): """POST a message with an Idempotency-Key (a fresh one unless `key` is given).""" h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})} if raw is not None: h.setdefault("Content-Type", "application/json") return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"], json_body=body if raw is None else None, raw=raw, headers=h, schema="Message" if expected == 201 else None, **kwargs) def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: return self.post_message(actor, chat, {"text": text}, key=key).json() def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png", filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None): """POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string (to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked).""" h = dict(headers or {}) if content_type is not None: h["Content-Type"] = content_type if query is None: query = "" if filename is None else "filename=" + quote(filename, safe="") path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "") if isinstance(data, (bytes, bytearray)): return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h, schema="Attachment" if expected == 201 else None) h["Authorization"] = "Bearer " + actor["token"] response = self.client.post(path, content=data, headers=h) assert response.status_code == expected, (response.status_code, response.text[:300]) return response def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: """Walk all pages; returns (items, number_of_pages).""" items, pages, cursor = [], 0, None while True: query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "") page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200, token=actor["token"], schema="MessagePage").json() pages += 1 items.extend(page["items"]) cursor = page["next_cursor"] if cursor is None: return items, pages assert len(page["items"]) == limit, "a non-final page must be full" assert len(items) < 100_000, "cursor loop" def _stand_is_up() -> bool: try: return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200 except httpx.HTTPError: return False def pytest_collection_modifyitems(config, items): if any("stack" in item.keywords for item in items) and not _stand_is_up(): skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait") for item in items: if "stack" in item.keywords: item.add_marker(skip) @pytest.fixture(scope="session") def api() -> Api: return Api() def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess: """Run scripts/ (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers.""" script = str(ROOT / "scripts" / name) if os.name == "nt": command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script] else: command = [script] return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True, timeout=timeout, env={**os.environ, **env}) def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str: """Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3 credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is not published on the host; this is how the tests inject faults directly into the bucket.""" script = ("import asyncio, base64, json, sys\n" "from messenger.config import Settings, s3_settings\n" "from messenger.storage import Storage, ObjectMissing\n" "async def main():\n" " st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n" " s = Storage(st)\n" " await s.open()\n" " try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) + " finally:\n" " await s.close()\n" "asyncio.run(main())\n") if service == "api": cmd = ["exec", "-T", "api", "python", "-c", script] else: cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script] return compose(*cmd, timeout=120, stdin=stdin).stdout.strip() def s3_put(key: str, data: bytes) -> None: """Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB).""" s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", stdin=base64.b64encode(data).decode()) def s3_delete(key: str) -> None: s3_exec(f"await s.delete({key!r})") def s3_size(key: str) -> int | None: out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)") size = int(out.splitlines()[-1]) return None if size < 0 else size def s3_keys() -> set[str]: """Every object key under o/ (needs List: runs with the reconciler's key pair).""" out = s3_exec("keys, after = [], None\n" "while True:\n" " page, more = await s.list_page('o/', after, 1000)\n" " keys += [o.key for o in page]\n" " if not more or not page:\n" " break\n" " after = page[-1].key\n" "print(json.dumps(keys))", service="reconciler") return set(json.loads(out.splitlines()[-1])) def reconcile(*args: str, expect_code: int = 0) -> dict: """Run the reconciler CLI (compose service, profile tools) and return its JSON report.""" # --no-deps: never let `compose run` start a store that a test has stopped on purpose. result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False, timeout=300) assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:]) return json.loads(result.stdout) def findings_by_key(report: dict) -> dict[str, dict]: return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items} def attachment_row(attachment_id: str) -> dict | None: out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, " "'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id) return json.loads(out) if out else None def probe_inside(replica: dict, path: str) -> int: """HTTP status of `path` asked inside the replica container itself (bypassing the LB).""" code = ("import urllib.request as u, urllib.error as e\n" "try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n" "except e.HTTPError as x: print(x.code)") % path out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True, timeout=30).stdout.strip() return int(out or 0)