Files
mtusi2026lab4/student-tests/conftest.py
T
Alexey BodanyukandClaude Opus 5.5 a2d867bec6 lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:05:03 +03:00

381 lines
18 KiB
Python

"""Shared helpers for the student tests.
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
`docker compose` for database checks (psql inside the postgres container), the HAProxy
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
python -m pytest student-tests # everything (stand must be up)
python -m pytest student-tests -m "not stack" # unit only
"""
from __future__ import annotations
import base64
import contextlib
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import quote
import httpx2 as httpx
import pytest
from jsonschema import Draft202012Validator
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
import lbctl # noqa: E402
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
def pytest_configure(config):
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
def schema_errors(value, name: str) -> list[str]:
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
return [e.message for e in validator.iter_errors(value)]
def basic(username: str, password: str) -> str:
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
# ------------------------------------------------------------------ docker / psql
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", check=check, timeout=timeout, input=stdin)
def psql(sql: str) -> str:
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
return result.stdout.strip()
def wait_ready(timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.3)
raise AssertionError("stand did not become ready")
def wait_healthy(*services: str, timeout: float = 120) -> None:
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
deadline = time.monotonic() + timeout
pending = set(services)
while pending and time.monotonic() < deadline:
for service in list(pending):
ids = compose("ps", "-q", service, check=False).stdout.split()
if ids:
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
capture_output=True, text=True).stdout.split()
if states and all(state == "healthy" for state in states):
pending.discard(service)
time.sleep(0.5)
assert not pending, f"not healthy: {pending}"
def wait_stand(replicas: int = REPLICAS) -> None:
"""Readiness through the LB AND every replica back in HAProxy rotation."""
wait_ready()
lbctl.wait_up(replicas, timeout=90)
def restart_services(*services: str) -> None:
compose("restart", *services)
wait_healthy(*services)
wait_stand()
@contextlib.contextmanager
def service_down(service: str, how: str = "stop"):
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
assert how in {"stop", "pause"}
compose(how, service)
try:
yield
finally:
compose("unpause" if how == "pause" else "start", service)
wait_healthy(service)
wait_stand()
@contextlib.contextmanager
def api_with_env(**overrides: str):
"""Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2),
then ALWAYS recreate it with the normal configuration."""
env = {**os.environ, **overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
try:
yield
finally:
clean = {k: v for k, v in os.environ.items() if k not in overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
def running_replicas() -> list[dict]:
return [r for r in lbctl.replicas() if r["state"] == "running"]
@contextlib.contextmanager
def replica_stopped(replica: dict, mode: str = "stop"):
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
and wait until HAProxy has every replica UP."""
assert mode in {"stop", "kill"}
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
capture_output=True, text=True, check=True, timeout=60)
try:
yield
finally:
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
wait_healthy("api")
wait_stand()
def api_env(name: str) -> str:
"""Current value of an environment variable inside the running api container."""
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
# ------------------------------------------------------------------ HTTP
class Api:
"""HTTP helper that checks the invariants every response must satisfy."""
def __init__(self, base_url: str = BASE_URL):
self.client = httpx.Client(base_url=base_url, timeout=15)
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
lb_error_ok: bool = False):
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
accepted without X-Instance-Id; every response of an API replica must carry it."""
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
if json_body is not None:
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
response = self.client.request(method, path, content=raw, headers=h)
allowed = (expected,) if isinstance(expected, int) else expected
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
request_id = response.headers.get("x-request-id")
assert request_id, "every response must carry X-Request-Id"
if not response.headers.get("x-instance-id"):
assert lb_error_ok and response.status_code in (502, 503, 504), \
("missing X-Instance-Id", method, path, response.status_code)
if response.status_code >= 400:
body = response.json()
assert schema_errors(body, "Error") == [], response.text
assert body["error"]["request_id"] == request_id
assert "Traceback" not in response.text and "psycopg" not in response.text
if response.status_code == 401:
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
elif schema:
assert schema_errors(response.json(), schema) == [], response.text
return response
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
return {"id": user["id"], "user": user, "username": username, "password": password}
def login(self, actor: dict) -> str:
session = self.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])},
schema="Session").json()
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
return session["token"]
def user(self, label: str = "u", **kwargs) -> dict:
actor = self.register(label, **kwargs)
actor["token"] = self.login(actor)
return actor
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
ids = [owner["id"], *(m["id"] for m in members)]
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
raw: bytes | None = None, headers: dict | None = None, **kwargs):
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
if raw is not None:
h.setdefault("Content-Type", "application/json")
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
json_body=body if raw is None else None, raw=raw, headers=h,
schema="Message" if expected == 201 else None, **kwargs)
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
return self.post_message(actor, chat, {"text": text}, key=key).json()
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
h = dict(headers or {})
if content_type is not None:
h["Content-Type"] = content_type
if query is None:
query = "" if filename is None else "filename=" + quote(filename, safe="")
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
if isinstance(data, (bytes, bytearray)):
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
schema="Attachment" if expected == 201 else None)
h["Authorization"] = "Bearer " + actor["token"]
response = self.client.post(path, content=data, headers=h)
assert response.status_code == expected, (response.status_code, response.text[:300])
return response
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
items, pages, cursor = [], 0, None
while True:
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
token=actor["token"], schema="MessagePage").json()
pages += 1
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items, pages
assert len(page["items"]) == limit, "a non-final page must be full"
assert len(items) < 100_000, "cursor loop"
def _stand_is_up() -> bool:
try:
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
except httpx.HTTPError:
return False
def pytest_collection_modifyitems(config, items):
if any("stack" in item.keywords for item in items) and not _stand_is_up():
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
for item in items:
if "stack" in item.keywords:
item.add_marker(skip)
@pytest.fixture(scope="session")
def api() -> Api:
return Api()
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
script = str(ROOT / "scripts" / name)
if os.name == "nt":
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
else:
command = [script]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
timeout=timeout, env={**os.environ, **env})
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
not published on the host; this is how the tests inject faults directly into the bucket."""
script = ("import asyncio, base64, json, sys\n"
"from messenger.config import Settings, s3_settings\n"
"from messenger.storage import Storage, ObjectMissing\n"
"async def main():\n"
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
" s = Storage(st)\n"
" await s.open()\n"
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
" finally:\n"
" await s.close()\n"
"asyncio.run(main())\n")
if service == "api":
cmd = ["exec", "-T", "api", "python", "-c", script]
else:
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
def s3_put(key: str, data: bytes) -> None:
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
stdin=base64.b64encode(data).decode())
def s3_delete(key: str) -> None:
s3_exec(f"await s.delete({key!r})")
def s3_size(key: str) -> int | None:
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
size = int(out.splitlines()[-1])
return None if size < 0 else size
def s3_keys() -> set[str]:
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
out = s3_exec("keys, after = [], None\n"
"while True:\n"
" page, more = await s.list_page('o/', after, 1000)\n"
" keys += [o.key for o in page]\n"
" if not more or not page:\n"
" break\n"
" after = page[-1].key\n"
"print(json.dumps(keys))", service="reconciler")
return set(json.loads(out.splitlines()[-1]))
def reconcile(*args: str, expect_code: int = 0) -> dict:
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
timeout=300)
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
return json.loads(result.stdout)
def findings_by_key(report: dict) -> dict[str, dict]:
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
def attachment_row(attachment_id: str) -> dict | None:
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
return json.loads(out) if out else None
def probe_inside(replica: dict, path: str) -> int:
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
code = ("import urllib.request as u, urllib.error as e\n"
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
"except e.HTTPError as x: print(x.code)") % path
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
timeout=30).stdout.strip()
return int(out or 0)