forked from mtusi2026/mtusi2026lab4
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key; dry-run read-only конструктивно (default_transaction_read_only + S3 без записи), --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо), перепроверка под FOR UPDATE, предохранитель от массового удаления; - attachment_ids хранится в строке сообщения (история остаётся одним index range scan), message_attachments — для FK и обратного поиска; - семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY); - тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки, приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler; - бенчмарки памяти загрузок и стоимости обхода reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
381 lines
18 KiB
Python
381 lines
18 KiB
Python
"""Shared helpers for the student tests.
|
|
|
|
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
|
|
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
|
|
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
|
|
`docker compose` for database checks (psql inside the postgres container), the HAProxy
|
|
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
|
|
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
|
|
|
|
python -m pytest student-tests # everything (stand must be up)
|
|
python -m pytest student-tests -m "not stack" # unit only
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import httpx2 as httpx
|
|
import pytest
|
|
from jsonschema import Draft202012Validator
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
import lbctl # noqa: E402
|
|
|
|
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
|
|
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
|
|
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
|
|
|
|
|
|
def pytest_configure(config):
|
|
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
|
|
|
|
|
|
def schema_errors(value, name: str) -> list[str]:
|
|
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
|
|
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
|
|
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
|
|
return [e.message for e in validator.iter_errors(value)]
|
|
|
|
|
|
def basic(username: str, password: str) -> str:
|
|
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
|
|
|
|
# ------------------------------------------------------------------ docker / psql
|
|
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
|
|
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8", check=check, timeout=timeout, input=stdin)
|
|
|
|
|
|
def psql(sql: str) -> str:
|
|
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
|
|
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
|
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def wait_ready(timeout: float = 90) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
|
|
return
|
|
except httpx.HTTPError:
|
|
pass
|
|
time.sleep(0.3)
|
|
raise AssertionError("stand did not become ready")
|
|
|
|
|
|
def wait_healthy(*services: str, timeout: float = 120) -> None:
|
|
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
|
|
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
|
|
deadline = time.monotonic() + timeout
|
|
pending = set(services)
|
|
while pending and time.monotonic() < deadline:
|
|
for service in list(pending):
|
|
ids = compose("ps", "-q", service, check=False).stdout.split()
|
|
if ids:
|
|
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
|
|
capture_output=True, text=True).stdout.split()
|
|
if states and all(state == "healthy" for state in states):
|
|
pending.discard(service)
|
|
time.sleep(0.5)
|
|
assert not pending, f"not healthy: {pending}"
|
|
|
|
|
|
def wait_stand(replicas: int = REPLICAS) -> None:
|
|
"""Readiness through the LB AND every replica back in HAProxy rotation."""
|
|
wait_ready()
|
|
lbctl.wait_up(replicas, timeout=90)
|
|
|
|
|
|
def restart_services(*services: str) -> None:
|
|
compose("restart", *services)
|
|
wait_healthy(*services)
|
|
wait_stand()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def service_down(service: str, how: str = "stop"):
|
|
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
|
|
assert how in {"stop", "pause"}
|
|
compose(how, service)
|
|
try:
|
|
yield
|
|
finally:
|
|
compose("unpause" if how == "pause" else "start", service)
|
|
wait_healthy(service)
|
|
wait_stand()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def api_with_env(**overrides: str):
|
|
"""Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2),
|
|
then ALWAYS recreate it with the normal configuration."""
|
|
env = {**os.environ, **overrides}
|
|
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
|
|
capture_output=True, text=True, check=True, timeout=180)
|
|
wait_stand()
|
|
try:
|
|
yield
|
|
finally:
|
|
clean = {k: v for k, v in os.environ.items() if k not in overrides}
|
|
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
|
|
capture_output=True, text=True, check=True, timeout=180)
|
|
wait_stand()
|
|
|
|
|
|
def running_replicas() -> list[dict]:
|
|
return [r for r in lbctl.replicas() if r["state"] == "running"]
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def replica_stopped(replica: dict, mode: str = "stop"):
|
|
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
|
|
and wait until HAProxy has every replica UP."""
|
|
assert mode in {"stop", "kill"}
|
|
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
|
|
capture_output=True, text=True, check=True, timeout=60)
|
|
try:
|
|
yield
|
|
finally:
|
|
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
|
|
wait_healthy("api")
|
|
wait_stand()
|
|
|
|
|
|
def api_env(name: str) -> str:
|
|
"""Current value of an environment variable inside the running api container."""
|
|
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
|
|
|
|
|
|
# ------------------------------------------------------------------ HTTP
|
|
class Api:
|
|
"""HTTP helper that checks the invariants every response must satisfy."""
|
|
|
|
def __init__(self, base_url: str = BASE_URL):
|
|
self.client = httpx.Client(base_url=base_url, timeout=15)
|
|
|
|
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
|
|
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
|
|
lb_error_ok: bool = False):
|
|
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
|
|
accepted without X-Instance-Id; every response of an API replica must carry it."""
|
|
h = dict(headers or {})
|
|
if token is not None:
|
|
h["Authorization"] = "Bearer " + token
|
|
if json_body is not None:
|
|
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
|
|
h.setdefault("Content-Type", "application/json")
|
|
response = self.client.request(method, path, content=raw, headers=h)
|
|
allowed = (expected,) if isinstance(expected, int) else expected
|
|
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
|
|
request_id = response.headers.get("x-request-id")
|
|
assert request_id, "every response must carry X-Request-Id"
|
|
if not response.headers.get("x-instance-id"):
|
|
assert lb_error_ok and response.status_code in (502, 503, 504), \
|
|
("missing X-Instance-Id", method, path, response.status_code)
|
|
if response.status_code >= 400:
|
|
body = response.json()
|
|
assert schema_errors(body, "Error") == [], response.text
|
|
assert body["error"]["request_id"] == request_id
|
|
assert "Traceback" not in response.text and "psycopg" not in response.text
|
|
if response.status_code == 401:
|
|
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
|
|
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
|
|
elif schema:
|
|
assert schema_errors(response.json(), schema) == [], response.text
|
|
return response
|
|
|
|
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
|
|
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
|
|
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
|
|
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
|
|
return {"id": user["id"], "user": user, "username": username, "password": password}
|
|
|
|
def login(self, actor: dict) -> str:
|
|
session = self.call("POST", "/api/v1/auth/sessions", 201,
|
|
headers={"Authorization": basic(actor["username"], actor["password"])},
|
|
schema="Session").json()
|
|
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
|
|
return session["token"]
|
|
|
|
def user(self, label: str = "u", **kwargs) -> dict:
|
|
actor = self.register(label, **kwargs)
|
|
actor["token"] = self.login(actor)
|
|
return actor
|
|
|
|
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
|
|
ids = [owner["id"], *(m["id"] for m in members)]
|
|
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
|
|
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
|
|
|
|
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
|
|
raw: bytes | None = None, headers: dict | None = None, **kwargs):
|
|
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
|
|
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
|
|
if raw is not None:
|
|
h.setdefault("Content-Type", "application/json")
|
|
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
|
|
json_body=body if raw is None else None, raw=raw, headers=h,
|
|
schema="Message" if expected == 201 else None, **kwargs)
|
|
|
|
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
|
|
return self.post_message(actor, chat, {"text": text}, key=key).json()
|
|
|
|
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
|
|
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
|
|
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
|
|
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
|
|
h = dict(headers or {})
|
|
if content_type is not None:
|
|
h["Content-Type"] = content_type
|
|
if query is None:
|
|
query = "" if filename is None else "filename=" + quote(filename, safe="")
|
|
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
|
|
if isinstance(data, (bytes, bytearray)):
|
|
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
|
|
schema="Attachment" if expected == 201 else None)
|
|
h["Authorization"] = "Bearer " + actor["token"]
|
|
response = self.client.post(path, content=data, headers=h)
|
|
assert response.status_code == expected, (response.status_code, response.text[:300])
|
|
return response
|
|
|
|
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
|
|
"""Walk all pages; returns (items, number_of_pages)."""
|
|
items, pages, cursor = [], 0, None
|
|
while True:
|
|
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
|
|
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
|
|
token=actor["token"], schema="MessagePage").json()
|
|
pages += 1
|
|
items.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
return items, pages
|
|
assert len(page["items"]) == limit, "a non-final page must be full"
|
|
assert len(items) < 100_000, "cursor loop"
|
|
|
|
|
|
def _stand_is_up() -> bool:
|
|
try:
|
|
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
|
|
except httpx.HTTPError:
|
|
return False
|
|
|
|
|
|
def pytest_collection_modifyitems(config, items):
|
|
if any("stack" in item.keywords for item in items) and not _stand_is_up():
|
|
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
|
|
for item in items:
|
|
if "stack" in item.keywords:
|
|
item.add_marker(skip)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def api() -> Api:
|
|
return Api()
|
|
|
|
|
|
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
|
|
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
|
|
script = str(ROOT / "scripts" / name)
|
|
if os.name == "nt":
|
|
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
|
|
else:
|
|
command = [script]
|
|
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
|
|
timeout=timeout, env={**os.environ, **env})
|
|
|
|
|
|
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
|
|
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
|
|
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
|
|
not published on the host; this is how the tests inject faults directly into the bucket."""
|
|
script = ("import asyncio, base64, json, sys\n"
|
|
"from messenger.config import Settings, s3_settings\n"
|
|
"from messenger.storage import Storage, ObjectMissing\n"
|
|
"async def main():\n"
|
|
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
|
|
" s = Storage(st)\n"
|
|
" await s.open()\n"
|
|
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
|
|
" finally:\n"
|
|
" await s.close()\n"
|
|
"asyncio.run(main())\n")
|
|
if service == "api":
|
|
cmd = ["exec", "-T", "api", "python", "-c", script]
|
|
else:
|
|
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
|
|
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
|
|
|
|
|
|
def s3_put(key: str, data: bytes) -> None:
|
|
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
|
|
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
|
|
stdin=base64.b64encode(data).decode())
|
|
|
|
|
|
def s3_delete(key: str) -> None:
|
|
s3_exec(f"await s.delete({key!r})")
|
|
|
|
|
|
def s3_size(key: str) -> int | None:
|
|
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
|
|
size = int(out.splitlines()[-1])
|
|
return None if size < 0 else size
|
|
|
|
|
|
def s3_keys() -> set[str]:
|
|
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
|
|
out = s3_exec("keys, after = [], None\n"
|
|
"while True:\n"
|
|
" page, more = await s.list_page('o/', after, 1000)\n"
|
|
" keys += [o.key for o in page]\n"
|
|
" if not more or not page:\n"
|
|
" break\n"
|
|
" after = page[-1].key\n"
|
|
"print(json.dumps(keys))", service="reconciler")
|
|
return set(json.loads(out.splitlines()[-1]))
|
|
|
|
|
|
def reconcile(*args: str, expect_code: int = 0) -> dict:
|
|
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
|
|
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
|
|
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
|
|
timeout=300)
|
|
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def findings_by_key(report: dict) -> dict[str, dict]:
|
|
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
|
|
|
|
|
|
def attachment_row(attachment_id: str) -> dict | None:
|
|
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
|
|
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
|
|
return json.loads(out) if out else None
|
|
|
|
|
|
def probe_inside(replica: dict, path: str) -> int:
|
|
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
|
|
code = ("import urllib.request as u, urllib.error as e\n"
|
|
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
|
|
"except e.HTTPError as x: print(x.code)") % path
|
|
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
|
|
timeout=30).stdout.strip()
|
|
return int(out or 0)
|