From 58bb301fce977920fb8936331594dbe700b4b92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 1 Oct 2026 21:17:29 +0300 Subject: [PATCH 1/9] chore: keep lab plans and notes local --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b5a71896..f101a4f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # Local/private state +/docs/plans/ +/LAB_NOTES.md .env .env.* !.env.example -- 2.54.0 From f295b716fea646a1b7f83411c297025a557743a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 11:23:41 +0300 Subject: [PATCH 2/9] Carry durable messenger API into replicated lab --- migrations/001_initial.sql | 36 +++++ migrations/002_sessions.sql | 7 + requirements.txt | 6 + scripts/migrate.py | 30 ++++ src/api.py | 86 ++++++++++++ src/auth.py | 43 ++++++ src/db.py | 38 +++++ src/main.py | 66 +++++++++ src/models.py | 60 ++++++++ src/server.py | 14 ++ src/store.py | 244 +++++++++++++++++++++++++++++++++ student-tests/test_auth.py | 174 +++++++++++++++++++++++ student-tests/test_database.py | 85 ++++++++++++ student-tests/test_instance.py | 54 ++++++++ 14 files changed, 943 insertions(+) create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_sessions.sql create mode 100644 requirements.txt create mode 100644 scripts/migrate.py create mode 100644 src/api.py create mode 100644 src/auth.py create mode 100644 src/db.py create mode 100644 src/main.py create mode 100644 src/models.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_database.py create mode 100644 student-tests/test_instance.py diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..dbfb0065 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX chats_history_idx ON chats (created_at, id); + +CREATE TABLE memberships ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql new file mode 100644 index 00000000..5d2f364a --- /dev/null +++ b/migrations/002_sessions.sql @@ -0,0 +1,7 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..58563cf5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.118.3 +pydantic==2.12.4 +uvicorn==0.38.0 +psycopg[binary,pool]==3.2.12 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..e346d4f1 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,30 @@ +"""Apply numbered SQL migrations once, atomically and under a DB lock.""" + +from pathlib import Path +import sys + +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url + + +def main() -> None: + migration_dir = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(2026, 2)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())" + ) + for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")): + version = int(path.name.split("_", 1)[0]) + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone(): + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..5dca24b8 --- /dev/null +++ b/src/api.py @@ -0,0 +1,86 @@ +"""Messenger HTTP routes.""" + +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from .auth import basic_credentials, bearer_token +from .models import CreateChat, CreateMessage, CreateUser +from .store import store + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + return store.ready() + + +@router.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + return store.register(body.username, body.display_name, body.password) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request, response: Response): + if await request.body(): + raise HTTPException(400) + username, password = basic_credentials(request.headers.get("authorization")) + response.headers["Cache-Control"] = "no-store" + return store.create_session(username, password) + + +def current_user(request: Request) -> dict: + return store.authenticated(bearer_token(request.headers.get("authorization"))) + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + store.revoke_session(bearer_token(request.headers.get("authorization"))) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def user_me(actor: dict = Depends(current_user)): + return actor + + +@router.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(current_user)): + return store.create_chat(actor["id"], body.title, body.member_ids) + + +@router.get("/api/v1/chats") +def list_chats( + limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) +): + if not limit.isdecimal() or not 1 <= int(limit) <= 100: + raise HTTPException(400) + return store.list_chats(actor["id"], int(limit), cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): + return store.chat(actor["id"], str(chat_id)) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: UUID, body: CreateMessage, actor: dict = Depends(current_user)): + return store.create_message(actor["id"], str(chat_id), body.text) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: UUID, + limit: str = Query("50"), + cursor: str | None = Query(None, min_length=1), + actor: dict = Depends(current_user), +): + if not limit.isdecimal() or not 1 <= int(limit) <= 100: + raise HTTPException(400) + return store.list_messages(actor["id"], str(chat_id), int(limit), cursor) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..5714b173 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,43 @@ +"""Parse HTTP credentials without exposing them in errors or logs.""" + +import base64 +import binascii +import hashlib +import re + +from fastapi import HTTPException + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z") + + +def unauthorized(challenge: str) -> HTTPException: + return HTTPException(401, headers={"WWW-Authenticate": challenge}) + + +def basic_credentials(header: str | None) -> tuple[str, str]: + if not header or not header.startswith("Basic "): + raise unauthorized(BASIC_CHALLENGE) + try: + raw = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise unauthorized(BASIC_CHALLENGE) from None + if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized(BASIC_CHALLENGE) + return username, password + + +def bearer_token(header: str | None) -> str: + if not header or not header.startswith("Bearer "): + raise unauthorized(BEARER_CHALLENGE) + token = header[7:] + if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token): + raise unauthorized(BEARER_CHALLENGE) + return token + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..832ff074 --- /dev/null +++ b/src/db.py @@ -0,0 +1,38 @@ +"""Bounded PostgreSQL pool shared by API storage and health checks.""" + +import os +import atexit +from functools import lru_cache +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg_pool import PoolTimeout + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "postgresql:///messenger") + + +@lru_cache(maxsize=1) +def pool() -> ConnectionPool: + result = ConnectionPool( + database_url(), min_size=0, max_size=8, timeout=5, + kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row}, + open=True, + ) + atexit.register(result.close) + return result + + +def ready() -> bool: + try: + with pool().connection() as conn: + conn.execute("SELECT 1 FROM users LIMIT 0") + return True + except (psycopg.Error, PoolTimeout, OSError): + return False diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..bcac75fd --- /dev/null +++ b/src/main.py @@ -0,0 +1,66 @@ +"""FastAPI entry point and contract-wide response handling.""" + +from uuid import uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .api import router + +app = FastAPI() +app.include_router(router) +INSTANCE_ID = uuid4().hex + +ERRORS = { + 400: ("invalid_request", "Неверный запрос"), + 401: ("unauthorized", "Требуется действительный пользователь"), + 404: ("not_found", "Объект не найден"), + 409: ("conflict", "Конфликт данных"), + 413: ("payload_too_large", "Слишком большой запрос"), + 415: ("unsupported_media_type", "Неподдерживаемый тип данных"), + 429: ("rate_limited", "Слишком много запросов"), + 503: ("unavailable", "Сервис недоступен"), +} + + +def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse: + code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка")) + request_id = request.state.request_id + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})}, + ) + + +@app.middleware("http") +async def request_contract(request: Request, call_next): + request.state.request_id = str(uuid4()) + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions": + media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") + if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): + return error_response(request, 415) + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + return error_response(request, 400) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error(request: Request, exc: RequestValidationError): + return error_response(request, 400) + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, exc: HTTPException): + return error_response(request, exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def unexpected_error(request: Request, exc: Exception): + return error_response(request, 500) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..30f5706a --- /dev/null +++ b/src/models.py @@ -0,0 +1,60 @@ +"""Request bodies from the lab 1 contract.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class StrictBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(StrictBody): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def nonblank_name(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank display name") + return value + + @field_validator("password") + @classmethod + def nonblank_password(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank password") + return value + + +class CreateChat(StrictBody): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def nonblank_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + @classmethod + def valid_members(cls, value: list[str]) -> list[str]: + members = [str(UUID(member_id)) for member_id in value] + if len(set(members)) != len(members): + raise ValueError("duplicate member") + return members + + +class CreateMessage(StrictBody): + text: str = Field(min_length=1, max_length=4000) + + @field_validator("text") + @classmethod + def nonblank_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank text") + return value diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..20fb9c9c --- /dev/null +++ b/src/server.py @@ -0,0 +1,14 @@ +"""Run the ASGI worker as the container's main process.""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + ) diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..14c6d63c --- /dev/null +++ b/src/store.py @@ -0,0 +1,244 @@ +"""PostgreSQL-backed users, chats and message history.""" + +import base64 +import binascii +import json +import os +import secrets +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException +from psycopg.errors import UniqueViolation +from psycopg_pool import PoolTimeout + +from . import db +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized + +password_hasher = PasswordHasher() + + +def timestamp(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(resource: str, item: dict) -> str: + value = [resource, item["created_at"], item["id"]] + return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=") + + +def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + value = json.loads(raw) + if not isinstance(value, list) or len(value) != 3 or value[0] != resource: + raise ValueError + date_text, item_id = value[1:] + if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str): + raise ValueError + date = datetime.fromisoformat(date_text[:-1] + "+00:00") + if date.utcoffset().total_seconds() != 0: + raise ValueError + return date, UUID(item_id) + except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError): + raise HTTPException(400) from None + + +@contextmanager +def connection(): + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + raise HTTPException(503) from None + + +def user_view(row: dict) -> dict: + return { + "id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": timestamp(row["created_at"]), + } + + +def chat_view(row: dict) -> dict: + return { + "id": str(row["id"]), "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": timestamp(row["created_at"]), + } + + +def message_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), "text": row["text"], + "created_at": timestamp(row["created_at"]), + } + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" + + +class Store: + def ready(self) -> dict: + if not db.ready(): + raise HTTPException(503) + return {"status": "ok"} + + def register(self, username: str, display_name: str, password: str) -> dict: + password_hash = password_hasher.hash(password) + with connection() as conn: + try: + row = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid4(), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise HTTPException(409) from None + return user_view(row) + + def user(self, user_id: str | None) -> dict: + if not user_id: + raise HTTPException(401) + try: + user_uuid = UUID(user_id) + except (ValueError, TypeError): + raise HTTPException(401) from None + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,) + ).fetchone() + if row is None: + raise HTTPException(401) + return user_view(row) + + def create_session(self, username: str, password: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + try: + valid = row is not None and password_hasher.verify(row["password_hash"], password) + except (VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expiry = conn.execute( + "INSERT INTO sessions (token_hash,user_id,expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at", + (token_hash(token), row["id"], ttl), + ).fetchone()["expires_at"] + return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)} + + def authenticated(self, token: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s " + "JOIN users u ON u.id=s.user_id " + "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_view(row) + + def revoke_session(self, token: str) -> None: + with connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + + def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict: + if actor_id not in member_ids: + raise HTTPException(400) + members = [UUID(member_id) for member_id in member_ids] + with connection() as conn: + count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"] + if count != len(members): + raise HTTPException(400) + row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone() + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)", + (row["id"], member_id, position), + ) + return chat_view({**row, "member_ids": members}) + + def chat(self, actor_id: str, chat_id: str) -> dict: + with connection() as conn: + row = conn.execute( + f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id " + "WHERE access.user_id=%s AND c.id=%s", + (UUID(actor_id), UUID(chat_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return chat_view(row) + + def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"chats:{actor_id}") + query = (f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s") + params = [UUID(actor_id)] + if after: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(query, params).fetchall() + items = [chat_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} + + def create_message(self, actor_id: str, chat_id: str, text: str) -> dict: + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + row = conn.execute( + "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " + "RETURNING id,chat_id,sender_id,text,created_at", + (uuid4(), chat_uuid, actor_uuid, text), + ).fetchone() + return message_view(row) + + def list_messages(self, actor_id: str, chat_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"messages:{chat_id}") + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + query = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s" + params = [chat_uuid] + if after: + query += " AND (created_at,id)>(%s,%s)" + params.extend(after) + query += " ORDER BY created_at,id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + rows = conn.execute(query, params).fetchall() + items = [message_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None} + + +store = Store() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..e9fcdf09 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,174 @@ +"""HTTP and database checks for durable Basic/Bearer sessions.""" + +import base64 +import hashlib +import json +import os +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.request +from uuid import uuid4 + +import psycopg + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class AuthTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + + @classmethod + def start_server(cls, ttl="3600"): + cls.instance_id = None + env = {**os.environ, "SESSION_TTL_SECONDS": ttl} + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API server exited during startup") + try: + cls.call("GET", "/health/ready", expected=200) + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + raise AssertionError("API server did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def tearDownClass(cls): + cls.stop_server() + + @classmethod + def call(cls, method, path, *, body=None, authorization=None, user_id=None, expected=200): + headers = {"Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode() + headers["Content-Type"] = "application/json" + if authorization is not None: + headers["Authorization"] = authorization + if user_id is not None: + headers["X-User-Id"] = user_id + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", data=data, headers=headers, method=method, + ) + try: + response = urllib.request.urlopen(request, timeout=5) + except urllib.error.HTTPError as error: + response = error + with response: + status = response.status + response_headers = response.headers + data = response.read() + assert status == expected, (method, path, expected, status) + request_id = response_headers.get("X-Request-Id") + assert request_id + instance_id = response_headers.get("X-Instance-Id") + assert instance_id + if cls.instance_id is None: + cls.instance_id = instance_id + else: + assert instance_id == cls.instance_id + payload = json.loads(data) if data else None + if status >= 400: + assert payload["error"]["request_id"] == request_id + assert "password" not in data.decode().lower() + assert "token" not in data.decode().lower() + return payload, response_headers + + def user(self, password="Пароль:очень-длинный-123"): + username = "u_" + uuid4().hex[:16] + body = {"username": username, "display_name": "Алиса", "password": password} + user, _ = self.call("POST", "/api/v1/users", body=body, expected=201) + self.assertNotIn("password", user) + return user, password + + def login(self, username, password, expected=201): + encoded = base64.b64encode(f"{username}:{password}".encode()).decode() + return self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, expected=expected) + + def test_basic_bearer_logout_and_storage(self): + alice, password = self.user() + bob, _ = self.user() + session, headers = self.login(alice["username"], password) + self.assertEqual(session["token_type"], "Bearer") + self.assertEqual(session["user"], alice) + self.assertGreaterEqual(len(base64.urlsafe_b64decode(session["token"] + "=" * (-len(session["token"]) % 4))), 32) + self.assertEqual(headers.get("Cache-Control"), "no-store") + bearer = "Bearer " + session["token"] + me, _ = self.call("GET", "/api/v1/users/me", authorization=bearer, user_id=bob["id"]) + self.assertEqual(me, alice) + with psycopg.connect(URL) as conn: + row = conn.execute("SELECT token_hash,expires_at FROM sessions WHERE user_id=%s", (alice["id"],)).fetchone() + password_hash = conn.execute("SELECT password_hash FROM users WHERE id=%s", (alice["id"],)).fetchone()[0] + self.assertEqual(row[0], hashlib.sha256(session["token"].encode()).digest()) + self.assertNotEqual(password_hash, password) + self.assertTrue(password_hash.startswith("$argon2id$")) + self.call("DELETE", "/api/v1/auth/sessions/current", authorization=bearer, expected=204) + for method, path in (("GET", "/api/v1/users/me"), ("DELETE", "/api/v1/auth/sessions/current")): + _, headers = self.call(method, path, authorization=bearer, expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + + def test_challenges_and_uniform_credentials(self): + user, password = self.user() + failures = [] + for username, candidate in ((user["username"], "wrong-password-123"), ("missing_user_123", password)): + payload, headers = self.login(username, candidate, expected=401) + failures.append((payload["error"]["code"], payload["error"]["message"])) + self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(failures[0], failures[1]) + encoded = base64.b64encode(f"{user['username']}:{password}".encode()).decode() + for authorization in (None, "Basic " + encoded, "Bearer garbage"): + _, headers = self.call("GET", "/api/v1/users/me", authorization=authorization, user_id=user["id"], expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + _, headers = self.call("POST", "/api/v1/auth/sessions", authorization="Bearer garbage", expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"') + self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, body={"unexpected": True}, expected=400) + + def test_z_short_ttl_survives_server_restart_without_extension(self): + self.stop_server() + self.start_server(ttl="2") + try: + user, password = self.user() + session, _ = self.login(user["username"], password) + bearer = "Bearer " + session["token"] + self.call("GET", "/api/v1/users/me", authorization=bearer) + with psycopg.connect(URL) as conn: + before = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0] + self.stop_server() + self.start_server(ttl="3600") + with psycopg.connect(URL) as conn: + after = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0] + self.assertEqual(before, after) + time.sleep(2.1) + _, headers = self.call("GET", "/api/v1/users/me", authorization=bearer, expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + finally: + self.stop_server() + self.start_server() + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_database.py b/student-tests/test_database.py new file mode 100644 index 00000000..27567af0 --- /dev/null +++ b/student-tests/test_database.py @@ -0,0 +1,85 @@ +"""Integration checks against a disposable PostgreSQL database.""" + +import os +import subprocess +import sys +import unittest +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import psycopg + +from src.store import Store + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class DatabaseTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + cls.store = Store() + + def user(self): + name = "u_" + uuid4().hex[:16] + return self.store.register(name, "Студент", "Test-password-123") + + def test_migration_and_unique_username_race(self): + with psycopg.connect(URL) as conn: + versions = conn.execute("SELECT version FROM schema_migrations").fetchall() + self.assertEqual(versions, [(1,), (2,)]) + name = "u_" + uuid4().hex[:16] + + def register(_): + try: + return self.store.register(name, "Студент", "Test-password-123") + except Exception as exc: + return exc + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(register, range(2))) + self.assertEqual(sum(isinstance(result, dict) for result in results), 1) + self.assertEqual([result.status_code for result in results if not isinstance(result, dict)], [409]) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1) + password_hash = conn.execute("SELECT password_hash FROM users WHERE username=%s", (name,)).fetchone()[0] + self.assertTrue(password_hash.startswith("$argon2id$")) + self.assertNotIn("password", str(next(result for result in results if isinstance(result, dict)))) + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1) + + def test_chat_rollback_and_keyset_history(self): + alice, bob, eve = self.user(), self.user(), self.user() + with psycopg.connect(URL) as conn: + before = conn.execute("SELECT count(*) FROM chats").fetchone()[0] + with self.assertRaises(Exception) as failure: + self.store.create_chat(alice["id"], "invalid", [alice["id"], str(uuid4())]) + self.assertEqual(failure.exception.status_code, 400) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM chats").fetchone()[0], before) + + chat = self.store.create_chat(alice["id"], "Valid", [alice["id"], bob["id"]]) + chat2 = self.store.create_chat(alice["id"], "Second", [alice["id"]]) + chat_page = self.store.list_chats(alice["id"], 1, None) + self.assertEqual(chat_page["items"], [chat]) + self.assertEqual(self.store.list_chats(alice["id"], 1, chat_page["next_cursor"]), {"items": [chat2], "next_cursor": None}) + first = self.store.create_message(alice["id"], chat["id"], "первое") + second = self.store.create_message(bob["id"], chat["id"], "второе") + page = self.store.list_messages(alice["id"], chat["id"], 1, None) + self.assertEqual(page["items"], [first]) + self.assertEqual(self.store.list_messages(bob["id"], chat["id"], 1, page["next_cursor"]), {"items": [second], "next_cursor": None}) + self.assertEqual(Store().chat(bob["id"], chat["id"]), chat) + with self.assertRaises(Exception) as wrong_cursor: + self.store.list_messages(alice["id"], chat2["id"], 1, page["next_cursor"]) + self.assertEqual(wrong_cursor.exception.status_code, 400) + with self.assertRaises(Exception) as hidden: + self.store.chat(eve["id"], chat["id"]) + self.assertEqual(hidden.exception.status_code, 404) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py new file mode 100644 index 00000000..87e14628 --- /dev/null +++ b/student-tests/test_instance.py @@ -0,0 +1,54 @@ +"""Every API response identifies the live replica process.""" + +import json +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.request + + +class InstanceHeaderTests(unittest.TestCase): + def test_process_alias_on_success_and_error(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(port)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + for _ in range(100): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/live", timeout=1) as response: + live_id = response.headers.get("X-Instance-Id") + self.assertEqual(json.load(response), {"status": "ok"}) + break + except urllib.error.URLError: + if server.poll() is not None: + self.fail("API exited before readiness") + time.sleep(0.05) + else: + self.fail("API did not start") + self.assertTrue(live_id) + for path in ("/health/live", "/no-such-route"): + try: + response = urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=1) + except urllib.error.HTTPError as error: + response = error + with response: + self.assertEqual(response.headers.get("X-Instance-Id"), live_id) + self.assertTrue(response.headers.get("X-Request-Id")) + finally: + server.terminate() + try: + server.wait(timeout=5) + except subprocess.TimeoutExpired: + server.kill() + server.wait(timeout=5) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 44fcce080f1c5014b77c3dc5f6ec8cfdcc0ff84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 11:36:23 +0300 Subject: [PATCH 3/9] Make message retries idempotent across replicas --- migrations/003_message_replays.sql | 11 +++++++ src/api.py | 8 +++-- src/store.py | 30 ++++++++++++++++- student-tests/test_database.py | 6 ++-- student-tests/test_idempotency.py | 53 ++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 migrations/003_message_replays.sql create mode 100644 student-tests/test_idempotency.py diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql new file mode 100644 index 00000000..e5a86270 --- /dev/null +++ b/migrations/003_message_replays.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_replays ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, request_key) +); + +CREATE INDEX message_replays_expiry_idx ON message_replays (created_at); diff --git a/src/api.py b/src/api.py index 5dca24b8..953a365c 100644 --- a/src/api.py +++ b/src/api.py @@ -1,5 +1,6 @@ """Messenger HTTP routes.""" +import re from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response @@ -70,8 +71,11 @@ def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): @router.post("/api/v1/chats/{chat_id}/messages", status_code=201) -def create_message(chat_id: UUID, body: CreateMessage, actor: dict = Depends(current_user)): - return store.create_message(actor["id"], str(chat_id), body.text) +def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): + key = request.headers.get("Idempotency-Key") + if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400) + return store.create_message(actor["id"], str(chat_id), body.text, key) @router.get("/api/v1/chats/{chat_id}/messages") diff --git a/src/store.py b/src/store.py index 14c6d63c..d91a5644 100644 --- a/src/store.py +++ b/src/store.py @@ -2,6 +2,7 @@ import base64 import binascii +import hashlib import json import os import secrets @@ -205,19 +206,46 @@ class Store: items = [chat_view(row) for row in rows[:limit]] return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} - def create_message(self, actor_id: str, chat_id: str, text: str) -> dict: + def create_message(self, actor_id: str, chat_id: str, text: str, key: str) -> dict: chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + request_hash = hashlib.sha256(json.dumps({"text": text}, ensure_ascii=False, sort_keys=True).encode()).digest() with connection() as conn: member = conn.execute( "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) ).fetchone() if member is None: raise HTTPException(404) + conn.execute( + "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s " + "AND created_at dict: diff --git a/student-tests/test_database.py b/student-tests/test_database.py index 27567af0..132318f9 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -30,7 +30,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,)]) + self.assertEqual(versions, [(1,), (2,), (3,)]) name = "u_" + uuid4().hex[:16] def register(_): @@ -67,8 +67,8 @@ class DatabaseTests(unittest.TestCase): chat_page = self.store.list_chats(alice["id"], 1, None) self.assertEqual(chat_page["items"], [chat]) self.assertEqual(self.store.list_chats(alice["id"], 1, chat_page["next_cursor"]), {"items": [chat2], "next_cursor": None}) - first = self.store.create_message(alice["id"], chat["id"], "первое") - second = self.store.create_message(bob["id"], chat["id"], "второе") + first = self.store.create_message(alice["id"], chat["id"], "первое", uuid4().hex) + second = self.store.create_message(bob["id"], chat["id"], "второе", uuid4().hex) page = self.store.list_messages(alice["id"], chat["id"], 1, None) self.assertEqual(page["items"], [first]) self.assertEqual(self.store.list_messages(bob["id"], chat["id"], 1, page["next_cursor"]), {"items": [second], "next_cursor": None}) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..dc52711b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,53 @@ +"""Concurrent PostgreSQL-backed message retries create one row.""" + +import os +import subprocess +import sys +import unittest +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import psycopg + +from src.store import Store + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class IdempotencyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + cls.store = Store() + + def test_parallel_replay_conflict_and_scope(self): + alice = self.store.register("u_" + uuid4().hex[:16], "Алиса", "Test-password-123") + bob = self.store.register("u_" + uuid4().hex[:16], "Боб", "Test-password-123") + chat = self.store.create_chat(alice["id"], "Test", [alice["id"], bob["id"]]) + other_chat = self.store.create_chat(alice["id"], "Other", [alice["id"]]) + key = uuid4().hex + + def send(_): + return Store().create_message(alice["id"], chat["id"], "same body", key) + + with ThreadPoolExecutor(max_workers=2) as pool: + first, replay = list(pool.map(send, range(2))) + self.assertEqual(first, replay) + with psycopg.connect(URL) as conn: + count = conn.execute( + "SELECT count(*) FROM messages WHERE chat_id=%s AND sender_id=%s AND text=%s", + (chat["id"], alice["id"], "same body"), + ).fetchone()[0] + self.assertEqual(count, 1) + + with self.assertRaises(Exception) as conflict: + self.store.create_message(alice["id"], chat["id"], "different body", key) + self.assertEqual(conflict.exception.status_code, 409) + self.assertNotEqual(self.store.create_message(bob["id"], chat["id"], "same body", key)["id"], first["id"]) + self.assertNotEqual(self.store.create_message(alice["id"], other_chat["id"], "same body", key)["id"], first["id"]) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 645164cb030ebdbc5f90b477bad2c4e4b3ad6883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 11:53:12 +0300 Subject: [PATCH 4/9] Run two API replicas behind HAProxy with verified failover --- .dockerignore | 2 + .env.example | 12 +----- Dockerfile | 12 ++++++ REPORT.md | 64 +++++++++++++++++++---------- compose.yaml | 70 ++++++++++++++++++++++++++++++++ evidence/failover-2026-10-02.txt | 40 ++++++++++++++++++ infra/haproxy.cfg | 26 ++++++++++++ scripts/restart.sh | 5 +++ scripts/stop-one.sh | 13 ++++++ 9 files changed, 211 insertions(+), 33 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/failover-2026-10-02.txt create mode 100644 infra/haproxy.cfg create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh diff --git a/.dockerignore b/.dockerignore index bc16e556..639a55ae 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ .git +docs/plans +LAB_NOTES.md .github .env .env.* diff --git a/.env.example b/.env.example index d32b9be0..ac22be39 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,3 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env; create secrets/postgres_password and secrets/database_url locally. APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1ecad6dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app +COPY --chown=app:app src/ src/ +COPY --chown=app:app migrations/ migrations/ +COPY --chown=app:app scripts/migrate.py scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["python", "-m", "src.server"] diff --git a/REPORT.md b/REPORT.md index c2679005..cd2d7d56 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,57 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Самохвалов Андрей Ремович, БПИ-2302. +- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 и исследование трека дополняются следующими блоками. +- Ветка: `solution/bpi2302-samohvalov`; commit/PR после публикации. +- Источники: `README.md`, `COURSE.md`, `contracts/`, публичные тесты репозитория, [HAProxy 3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). Код и текст подготовлены с помощью Codex; результаты ниже получены реальным запуском. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose, Python 3.9+ и `make`. Dockerfile собирает исходники в `python:3.12.12-slim-bookworm`; БД — `postgres:17.6-bookworm`, балансировщик — `haproxy:3.0.11-alpine`. Python-пакеты закреплены в `requirements.txt`. Стенд рассчитан на localhost и синтетические данные. + +```sh +cp .env.example .env +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +from secrets import token_urlsafe +password = token_urlsafe(32) +Path('secrets/postgres_password').write_text(password) +Path('secrets/database_url').write_text(f'postgresql://messenger:{password}@postgres:5432/messenger') +PY +chmod 600 .env secrets/* +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait --scale api=2 +make check +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env up -d --wait --scale api=2 # восстановить остановленную реплику +``` + +Для повторного failover после восстановления балансировщик должен обслуживать обе реплики; повторите команду и сверьте, какой контейнер остановлен. `docker compose --env-file .env ps -a` показывает реальный процесс. `migrate` — одноразовый сервис, который под advisory lock применяет SQL-миграции до запуска API; повторный `up` безопасно проверяет уже применённые версии. `docker compose --env-file .env down` сохраняет volume. Только для намеренного удаления учебных данных: `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy (`lb`, единственный опубликованный порт) → две FastAPI-реплики (`api`) → общий PostgreSQL (`postgres`). Сессии, сообщения и записи идемпотентности хранятся в PostgreSQL. Каждая реплика принимает любой действующий Bearer без sticky session. `X-Instance-Id` — случайный alias, постоянный в пределах процесса. Внешний порт PostgreSQL и API не опубликован. -## Доказательства критериев +Балансировка round-robin использует `server-template` на четыре слота, Docker DNS `127.0.0.11`, повторное разрешение адресов и `/health/ready` раз в 1 секунду (`fall 2`, `rise 1`). Сбой backend обнаруживается примерно за 2 проверки; проверка через публичный endpoint важнее расчётного интервала. HAProxy имеет connect timeout 2 с и client/server timeout 15 с. Compose ждёт готовности БД и однократной миграции; API имеет 10-секундный `stop_grace_period`, Uvicorn получает SIGTERM как главный процесс контейнера. На API установлены ограничения 0.5 CPU/256 MiB на реплику, на LB — 0.5 CPU/128 MiB. Дополнительное исследование фиксированного суммарного бюджета выполняется позже. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Создание чата и членов атомарно. POST сообщения требует ключ `[A-Za-z0-9._:-]{1,128}`; PostgreSQL unique key `(user_id, chat_id, request_key)` резервируется в транзакции вместе с сообщением и ссылкой на результат. Конкурентный повтор ждёт исход первой транзакции. При одинаковом тексте возвращается тот же `201 Message`, при другом — `409`. Ключ другой пары пользователь/чат независим. Повтор проверяет Bearer и членство до чтения сохранённого результата. Записи старше 24 часов по тому же ключу удаляются перед новым запросом; массовая очистка исторических ключей пока не автоматизирована. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Проверки уровня 3 -## Эксперименты и отказы +| Критерий | Команда/наблюдение | Результат | +| --- | --- | --- | +| Контракт | `make check`; локальный `make test BASE_URL=http://127.0.0.1:8083` через LB | PASS; публичный lab3 smoke | +| Реальные реплики | 12 GET `/health/live` через LB, `docker compose ps -a` | Два разных `X-Instance-Id`, два healthy API-контейнера; только LB опубликован наружу | +| Конкурентная идемпотентность | `python -m unittest discover -s student-tests` на временной PostgreSQL | 7 тестов PASS; два конкурентных вызова, одна запись `messages` | +| Полный рестарт | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS, прежние данные и токен; времена `StartedAt` обеих API, LB и PostgreSQL изменились | +| Отказ API-1 | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; `docker compose ps -a` | PASS, 2 реплики до отказа, 112 записей, 1 переходная ошибка, первая успешная запись через 2.935 с; API-1 `Exited (0)` | +| Отказ API-2 | Восстановить API-1, сдвинуть round-robin одним GET, повторить failover | PASS, 2 реплики до отказа, 110 записей, 1 переходная ошибка, первая успешная запись через 3.985 с; API-2 `Exited (0)` | -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Сырые безопасные агрегаты и времена контейнеров: `evidence/failover-2026-10-02.txt`. После опытов обе реплики восстановлены. Проверки выполнялись на macOS/Colima arm64 с локальным портом 8083, поскольку 8080 занят стендом lab2. На чистом clone `.env.example` использует 8080. `make validate` локально не запускался: зависимость `openapi-spec-validator` недоступна из-за DNS к `files.pythonhosted.org`; обязательный контракт не менялся, штатный CI job `contracts.yml` выполняет validate. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Ограничения и дальнейшие измерения -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд имеет один Docker-host, один LB и один PostgreSQL: отказ узла или БД не переживает. `depends_on` гарантирует порядок запуска, а маршрутизацию выполняет HAProxy. Проверка failover измеряет краткие ошибки и время восстановления, но не RPS до/после и не rolling update; это и исследование 1/2/4 реплик на фиксированном бюджете с повторениями будут добавлены для уровня 4/5. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..7e762985 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,70 @@ +name: mtusi2026lab3 + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + scale: 2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 10s + cpus: 0.5 + mem_limit: 256m + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + +volumes: + postgres_data: diff --git a/evidence/failover-2026-10-02.txt b/evidence/failover-2026-10-02.txt new file mode 100644 index 00000000..85f443e3 --- /dev/null +++ b/evidence/failover-2026-10-02.txt @@ -0,0 +1,40 @@ +Lab 3 local Compose evidence, 2026-10-02, macOS/Colima arm64. +All public requests used HAProxy at http://127.0.0.1:8083; API and PostgreSQL +had no published host ports. The local .env and secrets are ignored by Git. + +docker compose --env-file .env config --quiet -> PASS +docker compose --env-file .env up -d --build --wait --scale api=2 -> + api-1 healthy, api-2 healthy, postgres healthy, migrate exited 0, lb running. +make check -> PASS +make test BASE_URL=http://127.0.0.1:8083 -> PASS: lab3 public API scenario. +12 GET /health/live through LB observed two X-Instance-Id values. + +Isolated PostgreSQL integration run before Compose: + python -m unittest discover -s student-tests -v -> 7 tests, OK. + Concurrent same-key requests returned identical Message; SQL count was 1. + +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8083 + API-1 stop: observed_instances_before=2, successful_writes=112, + failed_attempts=1, first_recovery_seconds=2.935, PASS. + docker compose ps -a: api-1 Exited (0), api-2 healthy, lb/postgres running. + Restored api-1 using docker compose up -d --wait --scale api=2. + A repeat selected api-1 again; it was not counted as the second-replica proof. + Restored api-1, sent one GET to advance round-robin, and repeated: + API-2 stop: observed_instances_before=2, successful_writes=110, + failed_attempts=1, first_recovery_seconds=3.985, PASS. + docker compose ps -a: api-2 Exited (0), api-1 healthy, lb/postgres running. + Restored api-2; both replicas healthy. + +Persistence hook proof, times UTC; named postgres_data volume preserved: + before: + api-1 2026-10-02T08:46:12.886332892Z + api-2 2026-10-02T08:48:59.563753865Z + lb 2026-10-02T08:42:41.675055235Z + postgres 2026-10-02T08:42:32.879514014Z + make test-persistence ACTION_SCRIPT=scripts/restart.sh + -> PASS: confirmed data and existing session survived. + after: + api-1 2026-10-02T08:49:48.469504932Z + api-2 2026-10-02T08:49:48.407940425Z + lb 2026-10-02T08:49:54.047109297Z + postgres 2026-10-02T08:49:45.282318374Z diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..40952ab9 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 15s + timeout server 15s + +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 2 + timeout retry 1s + hold valid 1s + +frontend public + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c9148ca0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api lb postgres +docker compose --env-file .env up -d --wait --scale api=2 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..99c65a25 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +for container in $(docker compose --env-file .env ps -q api); do + instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$container" >/dev/null + exit 0 + fi +done +echo "target API instance not found" >&2 +exit 1 -- 2.54.0 From a2021b4e25969095d66adf99f195b2cd3e5c7242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 11:58:34 +0300 Subject: [PATCH 5/9] Return client error for oversized pagination limits --- src/api.py | 18 ++++++++++++------ student-tests/test_api_limits.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 student-tests/test_api_limits.py diff --git a/src/api.py b/src/api.py index 953a365c..e9b7a8f8 100644 --- a/src/api.py +++ b/src/api.py @@ -40,6 +40,16 @@ def current_user(request: Request) -> dict: return store.authenticated(bearer_token(request.headers.get("authorization"))) +def parse_limit(value: str) -> int: + try: + number = int(value) if value.isdecimal() else 0 + except ValueError: + raise HTTPException(400) from None + if not 1 <= number <= 100: + raise HTTPException(400) + return number + + @router.delete("/api/v1/auth/sessions/current", status_code=204) def revoke_session(request: Request): store.revoke_session(bearer_token(request.headers.get("authorization"))) @@ -60,9 +70,7 @@ def create_chat(body: CreateChat, actor: dict = Depends(current_user)): def list_chats( limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) ): - if not limit.isdecimal() or not 1 <= int(limit) <= 100: - raise HTTPException(400) - return store.list_chats(actor["id"], int(limit), cursor) + return store.list_chats(actor["id"], parse_limit(limit), cursor) @router.get("/api/v1/chats/{chat_id}") @@ -85,6 +93,4 @@ def list_messages( cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user), ): - if not limit.isdecimal() or not 1 <= int(limit) <= 100: - raise HTTPException(400) - return store.list_messages(actor["id"], str(chat_id), int(limit), cursor) + return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor) diff --git a/student-tests/test_api_limits.py b/student-tests/test_api_limits.py new file mode 100644 index 00000000..4936d6f3 --- /dev/null +++ b/student-tests/test_api_limits.py @@ -0,0 +1,20 @@ +"""Invalid pagination limits must remain client errors.""" + +import unittest + +from fastapi import HTTPException + +from src.api import parse_limit + + +class LimitTests(unittest.TestCase): + def test_valid_and_extremely_long_limits(self): + self.assertEqual(parse_limit("100"), 100) + for value in ("0", "101", "-1", "x", "9" * 4301): + with self.assertRaises(HTTPException) as failure: + parse_limit(value) + self.assertEqual(failure.exception.status_code, 400) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 23ed63f5933e2af4b611e9695b0655b061aa8d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:03:54 +0300 Subject: [PATCH 6/9] Prove per-replica failover and rolling replacement under load --- REPORT.md | 35 +++- compose.yaml | 4 + evidence/failover-u4-first-load.json | 17 ++ evidence/failover-u4-first-target.json | 16 ++ evidence/failover-u4-first.txt | 31 +++ evidence/failover-u4-second-target.json | 16 ++ evidence/rolling-u4.json | 18 ++ evidence/topology-u4.txt | 17 ++ scripts/rolling-update.sh | 38 ++++ scripts/stop-one.sh | 2 +- student-tests/test_operations.py | 239 ++++++++++++++++++++++++ 11 files changed, 429 insertions(+), 4 deletions(-) create mode 100644 evidence/failover-u4-first-load.json create mode 100644 evidence/failover-u4-first-target.json create mode 100644 evidence/failover-u4-first.txt create mode 100644 evidence/failover-u4-second-target.json create mode 100644 evidence/rolling-u4.json create mode 100644 evidence/topology-u4.txt create mode 100755 scripts/rolling-update.sh create mode 100644 student-tests/test_operations.py diff --git a/REPORT.md b/REPORT.md index cd2d7d56..55d92527 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 3 - Автор: Самохвалов Андрей Ремович, БПИ-2302. -- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 и исследование трека дополняются следующими блоками. +- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 проверен ниже; исследование трека 1 выполняется отдельно. - Ветка: `solution/bpi2302-samohvalov`; commit/PR после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/`, публичные тесты репозитория, [HAProxy 3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). Код и текст подготовлены с помощью Codex; результаты ниже получены реальным запуском. @@ -35,7 +35,9 @@ docker compose --env-file .env up -d --wait --scale api=2 # восстанови Клиент → HAProxy (`lb`, единственный опубликованный порт) → две FastAPI-реплики (`api`) → общий PostgreSQL (`postgres`). Сессии, сообщения и записи идемпотентности хранятся в PostgreSQL. Каждая реплика принимает любой действующий Bearer без sticky session. `X-Instance-Id` — случайный alias, постоянный в пределах процесса. Внешний порт PostgreSQL и API не опубликован. -Балансировка round-robin использует `server-template` на четыре слота, Docker DNS `127.0.0.11`, повторное разрешение адресов и `/health/ready` раз в 1 секунду (`fall 2`, `rise 1`). Сбой backend обнаруживается примерно за 2 проверки; проверка через публичный endpoint важнее расчётного интервала. HAProxy имеет connect timeout 2 с и client/server timeout 15 с. Compose ждёт готовности БД и однократной миграции; API имеет 10-секундный `stop_grace_period`, Uvicorn получает SIGTERM как главный процесс контейнера. На API установлены ограничения 0.5 CPU/256 MiB на реплику, на LB — 0.5 CPU/128 MiB. Дополнительное исследование фиксированного суммарного бюджета выполняется позже. +Балансировка round-robin использует `server-template` на четыре слота, Docker DNS `127.0.0.11`, повторное разрешение адресов и `/health/ready` раз в 1 секунду (`fall 2`, `rise 1`). Сбой backend обнаруживается примерно за 2 проверки; проверка через публичный endpoint важнее расчётного интервала. HAProxy имеет connect timeout 2 с и client/server timeout 15 с. Compose ждёт готовности БД и однократной миграции; API имеет 10-секундный `stop_grace_period`, Uvicorn получает SIGTERM как главный процесс контейнера. На API установлены ограничения 0.5 CPU/256 MiB на реплику, на LB — 0.5 CPU/128 MiB; на PostgreSQL — 1 CPU/512 MiB, на миграцию — 0.5 CPU/128 MiB. Дополнительное исследование фиксированного суммарного бюджета выполняется позже. + +API и миграция получают только `database_url`, PostgreSQL — только пароль БД, LB — только конфигурацию HAProxy в read-only mount. `student-tests/test_operations.py` проверяет опубликованные порты, mounts, secrets и лимиты из раскрытого Compose-конфига; фактические лимиты контейнеров сняты через `docker inspect` в `evidence/topology-u4.txt`. Создание чата и членов атомарно. POST сообщения требует ключ `[A-Za-z0-9._:-]{1,128}`; PostgreSQL unique key `(user_id, chat_id, request_key)` резервируется в транзакции вместе с сообщением и ссылкой на результат. Конкурентный повтор ждёт исход первой транзакции. При одинаковом тексте возвращается тот же `201 Message`, при другом — `409`. Ключ другой пары пользователь/чат независим. Повтор проверяет Bearer и членство до чтения сохранённого результата. Записи старше 24 часов по тому же ключу удаляются перед новым запросом; массовая очистка исторических ключей пока не автоматизирована. @@ -52,6 +54,33 @@ docker compose --env-file .env up -d --wait --scale api=2 # восстанови Сырые безопасные агрегаты и времена контейнеров: `evidence/failover-2026-10-02.txt`. После опытов обе реплики восстановлены. Проверки выполнялись на macOS/Colima arm64 с локальным портом 8083, поскольку 8080 занят стендом lab2. На чистом clone `.env.example` использует 8080. `make validate` локально не запускался: зависимость `openapi-spec-validator` недоступна из-за DNS к `files.pythonhosted.org`; обязательный контракт не менялся, штатный CI job `contracts.yml` выполняет validate. +## Уровень 4: эксплуатация двух реплик + +На macOS arm64 с Colima/Docker Engine 29.5.2 и Compose 5.3.1 использован Docker VM с 2 CPU и 2.05 GB RAM. HAProxy опрашивает `/health/ready` каждую 1 с, исключает backend после двух неудачных проверок и возвращает после одной удачной; API healthcheck выполняется каждые 2 с с timeout 3 с. После SIGTERM Uvicorn прекращает приём, `docker stop -t 10` и `stop_grace_period: 10s` дают процессу одинаковый срок для завершения текущих запросов. Интервал проверок даёт ориентир около 2 с на обнаружение отказа, но не жёсткую гарантию под нагрузкой; в измерениях новые успешные чтения/записи появились за 2.4–2.7 с от начала action. + +Последовательность двух адресных отказов, с восстановлением первой реплики перед вторым: + +```sh +python3 student-tests/test_operations.py --failover-index 1 --base-url http://127.0.0.1:8080 > evidence/failover-u4-first-target.json +docker compose --env-file .env up -d --wait --scale api=2 +python3 student-tests/test_operations.py --failover-index 2 --base-url http://127.0.0.1:8080 > evidence/failover-u4-second-target.json +docker compose --env-file .env up -d --wait --scale api=2 +python3 student-tests/test_operations.py --rolling --base-url http://127.0.0.1:8080 > evidence/rolling-u4.json +``` + +В локальном опыте указан порт `8083` вместо `8080`. Каждый адресный сценарий 15 секунд непрерывно читает и пишет с одним исходным Bearer, повторяет неопределённый POST с прежним ключом, убеждается в сохранности подтверждённых сообщений и проверяет, что выбранный Docker-контейнер действительно `exited`. Скорость намеренно ограничена паузой 50 мс на каждый из двух потоков: это проверка доступности, а не измерение предельной ёмкости. + +| Воздействие | Успешные чтения/записи | Ошибки чтения/POST | Успешные RPS | Первое чтение/запись после action | +| --- | ---: | ---: | ---: | ---: | +| Остановлен `api-5` | 214 / 211 | 1 / 1 | 28.141 | 2.555 / 2.489 с | +| Восстановлен `api-5`, остановлен `api-6` | 210 / 209 | 1 / 1 | 27.653 | 2.660 / 2.737 с | + +Дополнительно `tests/load.py` с 4 клиентами и 20 с во время отказа дал 6170 успешных POST (307.662 RPS), 5 ошибок, p50/p95/p99 успешных запросов 4.797/47.602/75.875 мс. Одновременно публичный `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` зафиксировал 112 успешных записей, 1 переходную ошибку, первый успех через 3.623 с и сохранил прежнюю сессию и данные. Эти два генератора имели разные чаты и независимые токены; агрегаты лежат в `evidence/failover-u4-first-load.json` и `evidence/failover-u4-first.txt`. + +`scripts/rolling-update.sh` поочерёдно останавливает и удаляет две исходные API-реплики, создавая и дожидаясь готовности замены после каждого шага; `--no-recreate` сохраняет другую работающую реплику. Под параллельной нагрузкой за 24.914 с прошло 7212 чтений, 4261 обычная запись и 302 пары одновременных POST с одинаковым ключом; 151 пара прошла через разные процессы. Было 2 ошибки чтения, 2 ошибки обычных POST и 2 попытки в парах с ошибкой. Все подтверждённые сообщения прочитаны обратно, дубли после неопределённых запросов не обнаружены; наблюдались четыре разных `X-Instance-Id`. После замены `api-5` и `api-6` были healthy, а публичный `make test` повторно прошёл. Метрики и ID реально заменённых контейнеров: `evidence/rolling-u4.json` и `evidence/topology-u4.txt`. + +Это **поочерёдная замена той же версии образа**: ни совместимость двух разных бинарных версий, ни миграция схемы во время нагрузки не проверялись. `migrate` выполняется до старта API, а скрипт замены запускает только API без повторной миграции. Для будущей смены схемы нужна отдельная совместимая миграция; данный опыт её не доказывает. + ## Ограничения и дальнейшие измерения -Стенд имеет один Docker-host, один LB и один PostgreSQL: отказ узла или БД не переживает. `depends_on` гарантирует порядок запуска, а маршрутизацию выполняет HAProxy. Проверка failover измеряет краткие ошибки и время восстановления, но не RPS до/после и не rolling update; это и исследование 1/2/4 реплик на фиксированном бюджете с повторениями будут добавлены для уровня 4/5. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. +Стенд имеет один Docker-host, один LB и один PostgreSQL: отказ узла или БД не переживает. `depends_on` гарантирует порядок запуска, а маршрутизацию выполняет HAProxy. Сценарии U4 показывают работоспособность и ошибки во время замены, но не предельную RPS при 1/2/4 репликах; отдельный эксперимент трека 1 будет добавлен для уровня 5. Генератор `tests/load.py` замкнутый и не устраняет coordinated omission. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. diff --git a/compose.yaml b/compose.yaml index 7e762985..de5bcff1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,6 +14,8 @@ services: interval: 2s timeout: 2s retries: 20 + cpus: 1.0 + mem_limit: 512m migrate: build: . @@ -25,6 +27,8 @@ services: postgres: condition: service_healthy restart: "no" + cpus: 0.5 + mem_limit: 128m api: build: . diff --git a/evidence/failover-u4-first-load.json b/evidence/failover-u4-first-load.json new file mode 100644 index 00000000..c1172439 --- /dev/null +++ b/evidence/failover-u4-first-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.054, + "concurrency": 4, + "successful_requests": 6170, + "failed_requests": 5, + "error_ratio": 0.00081, + "successful_rps": 307.662, + "successful_latency_ms": { + "p50": 4.797, + "p95": 47.602, + "p99": 75.875 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/failover-u4-first-target.json b/evidence/failover-u4-first-target.json new file mode 100644 index 00000000..bd9244fa --- /dev/null +++ b/evidence/failover-u4-first-target.json @@ -0,0 +1,16 @@ +{ + "action": "stop one selected API replica", + "target_container": "mtusi2026lab3-api-5", + "target_container_id": "48b688fc10cbbf4a6e08c459308198ce08c200437b4da35fe4214a1cd4c65a15", + "target_state_after_hook": "exited", + "elapsed_seconds": 15.103, + "stop_hook_seconds": 0.855, + "successful_reads": 214, + "failed_reads": 1, + "successful_writes": 211, + "failed_write_attempts": 1, + "successful_rps": 28.141, + "first_read_after_action_seconds": 2.555, + "first_write_after_action_seconds": 2.489, + "confirmed_messages_read_back": true +} diff --git a/evidence/failover-u4-first.txt b/evidence/failover-u4-first.txt new file mode 100644 index 00000000..866b821a --- /dev/null +++ b/evidence/failover-u4-first.txt @@ -0,0 +1,31 @@ +Commands: tests/load.py --duration 20 --concurrency 4, make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Failover exit: 0 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8083" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 3.623} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Load exit: 0 +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.054, + "concurrency": 4, + "successful_requests": 6170, + "failed_requests": 5, + "error_ratio": 0.00081, + "successful_rps": 307.662, + "successful_latency_ms": { + "p50": 4.797, + "p95": 47.602, + "p99": 75.875 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +Actual containers after hook: +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 10 minutes ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 9 minutes ago Exited (0) 16 seconds ago +mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 23 minutes 127.0.0.1:8083->8080/tcp +mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 18 minutes ago Exited (0) 57 seconds ago +mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 18 minutes ago Up 18 minutes (healthy) 5432/tcp diff --git a/evidence/failover-u4-second-target.json b/evidence/failover-u4-second-target.json new file mode 100644 index 00000000..4bc6ca3c --- /dev/null +++ b/evidence/failover-u4-second-target.json @@ -0,0 +1,16 @@ +{ + "action": "stop one selected API replica", + "target_container": "mtusi2026lab3-api-6", + "target_container_id": "bef5f29c9a9d87f8cf60e49ecfd8edff762ba156184fb413d6008f81750b4f00", + "target_state_after_hook": "exited", + "elapsed_seconds": 15.152, + "stop_hook_seconds": 0.864, + "successful_reads": 210, + "failed_reads": 1, + "successful_writes": 209, + "failed_write_attempts": 1, + "successful_rps": 27.653, + "first_read_after_action_seconds": 2.66, + "first_write_after_action_seconds": 2.737, + "confirmed_messages_read_back": true +} diff --git a/evidence/rolling-u4.json b/evidence/rolling-u4.json new file mode 100644 index 00000000..8fe4a2d8 --- /dev/null +++ b/evidence/rolling-u4.json @@ -0,0 +1,18 @@ +{ + "action": "same-version sequential API replacement", + "elapsed_seconds": 24.914, + "successful_reads": 7212, + "failed_reads": 2, + "successful_writes": 4261, + "failed_write_attempts": 2, + "successful_idempotent_pairs": 302, + "failed_pair_attempts": 2, + "cross_replica_pairs": 151, + "successful_rps": 484.749, + "observed_instance_count": 4, + "replacement_output": [ + "replaced API container bcd282fe6bdad720070e51cafc27b7084f647da7920c36e9f01e7fe56768dde7; two healthy replicas now serve traffic", + "replaced API container 797d654ce2c993fcdc74628443ea8deb6b2e5eafe4b54940ab06ae273976c890; two healthy replicas now serve traffic" + ], + "confirmed_messages_read_back": true +} diff --git a/evidence/topology-u4.txt b/evidence/topology-u4.txt new file mode 100644 index 00000000..e403a36c --- /dev/null +++ b/evidence/topology-u4.txt @@ -0,0 +1,17 @@ +Environment: Darwin arm64 +Docker Engine: 29.5.2 +Compose: 5.3.1 +Docker VM CPUs/MemTotal: 2 / 2054832128 bytes +Actual container limits (NanoCpus, Memory bytes): +/mtusi2026lab3-api-5 500000000 268435456 running +/mtusi2026lab3-api-6 500000000 268435456 running +/mtusi2026lab3-lb-1 500000000 134217728 running +/mtusi2026lab3-postgres-1 1000000000 536870912 running +/mtusi2026lab3-migrate-1 500000000 134217728 exited +Network/port and health observation: +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 15 minutes 127.0.0.1:8083->8080/tcp +mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 10 minutes ago Exited (0) 3 minutes ago +mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 10 minutes ago Up 10 minutes (healthy) 5432/tcp diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..09ca7d2b --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Replace the two original API processes one at a time. No schema changes here. +set -eu +cd "$(dirname "$0")/.." + +original=$(docker compose --env-file .env ps -q api) +set -- $original +if [ "$#" -ne 2 ]; then + echo "expected exactly two running API replicas" >&2 + exit 1 +fi + +for container in "$@"; do + if [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" != healthy ]; then + echo "API replica is not healthy before replacement" >&2 + exit 1 + fi +done + +for container in "$@"; do + docker stop -t 10 "$container" >/dev/null + docker rm "$container" >/dev/null + # --no-recreate preserves the other old replica while Compose fills the gap. + docker compose --env-file .env up -d --no-deps --no-recreate --wait --scale api=2 api >/dev/null + current=$(docker compose --env-file .env ps -q api) + set -- $current + if [ "$#" -ne 2 ]; then + echo "replacement did not restore two API replicas" >&2 + exit 1 + fi + for running in "$@"; do + if [ "$(docker inspect -f '{{.State.Health.Status}}' "$running")" != healthy ]; then + echo "replacement API is not healthy" >&2 + exit 1 + fi + done + echo "replaced API container $container; two healthy replicas now serve traffic" +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 99c65a25..86a70dac 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/.." for container in $(docker compose --env-file .env ps -q api); do instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then - docker stop -t 5 "$container" >/dev/null + docker stop -t 10 "$container" >/dev/null exit 0 fi done diff --git a/student-tests/test_operations.py b/student-tests/test_operations.py new file mode 100644 index 00000000..e545189e --- /dev/null +++ b/student-tests/test_operations.py @@ -0,0 +1,239 @@ +"""Operator checks: topology unittest plus explicit rolling replacement exercise.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import unittest +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 +from resilience import history # noqa: E402 + + +class TopologyTests(unittest.TestCase): + def test_limits_ports_mounts_and_secrets(self): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env.example", "config", "--format", "json"], + cwd=ROOT, capture_output=True, text=True, check=True, + ) + services = json.loads(proc.stdout)["services"] + self.assertEqual(set(services), {"postgres", "migrate", "api", "lb"}) + for name, service in services.items(): + self.assertNotIn("container_name", service, name) + self.assertGreater(float(service.get("cpus", 0)), 0, name) + self.assertGreater(int(service.get("mem_limit", 0)), 0, name) + if name != "lb": + self.assertFalse(service.get("ports"), name) + self.assertEqual(len(services["lb"]["ports"]), 1) + self.assertEqual(services["api"].get("volumes", []), []) + self.assertEqual(services["migrate"].get("volumes", []), []) + self.assertEqual(services["lb"].get("secrets", []), []) + self.assertEqual(len(services["lb"].get("volumes", [])), 1) + self.assertTrue(services["lb"]["volumes"][0]["read_only"]) + self.assertEqual({s["source"] for s in services["api"]["secrets"]}, {"database_url"}) + self.assertEqual({s["source"] for s in services["migrate"]["secrets"]}, {"database_url"}) + self.assertEqual({s["source"] for s in services["postgres"]["secrets"]}, {"postgres_password"}) + + +def rolling(base_url): + client = Client(base_url, None, timeout=2) + alice, bob, _, chat = client.bootstrap() + script = ROOT / "scripts/rolling-update.sh" + started = time.monotonic() + stop = False + + def writes(): + local = Client(base_url, None, timeout=2) + confirmed, errors = [], 0 + key, body = uuid4().hex, "rolling-" + uuid4().hex + while not stop: + try: + msg = local.message(alice, chat, body, key=key)["json"] + confirmed.append(msg) + key, body = uuid4().hex, "rolling-" + uuid4().hex + except Failure: + errors += 1 # Retry the uncertain write with its original key/body. + return confirmed, errors + + def reads(): + local = Client(base_url, None, timeout=2) + successes, errors, instances = 0, 0, set() + while not stop: + try: + result = local.call("GET", "/api/v1/users/me", actor=alice, schema="User") + successes += 1 + instances.add(result["headers"]["X-Instance-Id"]) + except Failure: + errors += 1 + return successes, errors, instances + + def duplicates(): + local = Client(base_url, None, timeout=2) + ids, attempted, errors, cross_replica = [], [], 0, 0 + with ThreadPoolExecutor(max_workers=2) as pair_pool: + while not stop: + key, body = uuid4().hex, "duplicate-" + uuid4().hex + attempted.append(body) + def send(_): + return local.message(alice, chat, body, key=key) + futures = [pair_pool.submit(send, index) for index in range(2)] + pair = [] + for future in futures: + try: + pair.append(future.result()) + except Failure: + errors += 1 + if len(pair) == 2: + assert pair[0]["json"]["id"] == pair[1]["json"]["id"] + ids.append(pair[0]["json"]["id"]) + cross_replica += pair[0]["headers"]["X-Instance-Id"] != pair[1]["headers"]["X-Instance-Id"] + time.sleep(0.05) + return ids, attempted, errors, cross_replica + + with ThreadPoolExecutor(max_workers=3) as pool: + write_result = pool.submit(writes) + read_result = pool.submit(reads) + duplicate_result = pool.submit(duplicates) + proc = subprocess.Popen([str(script)], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + stdout, stderr = proc.communicate(timeout=90) + if proc.returncode: + raise RuntimeError("rolling replacement failed: " + stderr.strip()) + time.sleep(3) # Include DNS/LB recovery after the last replacement. + finally: + stop = True + if proc.poll() is None: + proc.terminate() + proc.wait(timeout=5) + confirmed, write_errors = write_result.result() + read_count, read_errors, instances = read_result.result() + duplicate_ids, attempted_duplicates, duplicate_errors, cross_replica = duplicate_result.result() + + elapsed = time.monotonic() - started + messages = history(client, bob, chat) + seen = {message["id"] for message in messages} + assert confirmed and read_count and duplicate_ids, "load did not exercise all operations" + assert {message["id"] for message in confirmed}.issubset(seen), "confirmed write missing" + assert set(duplicate_ids).issubset(seen), "idempotent write missing" + duplicate_counts = {body: 0 for body in attempted_duplicates} + for message in messages: + if message["text"] in duplicate_counts: + duplicate_counts[message["text"]] += 1 + assert all(count <= 1 for count in duplicate_counts.values()), "uncertain retry duplicated a message" + assert len(instances) >= 2 and cross_replica > 0, "rolling test did not cross replicas" + print(json.dumps({ + "action": "same-version sequential API replacement", "elapsed_seconds": round(elapsed, 3), + "successful_reads": read_count, "failed_reads": read_errors, + "successful_writes": len(confirmed), "failed_write_attempts": write_errors, + "successful_idempotent_pairs": len(duplicate_ids), "failed_pair_attempts": duplicate_errors, + "cross_replica_pairs": cross_replica, + "successful_rps": round((read_count + len(confirmed) + 2 * len(duplicate_ids)) / elapsed, 3), + "observed_instance_count": len(instances), "replacement_output": stdout.strip().splitlines(), + "confirmed_messages_read_back": True, + }, ensure_ascii=False, indent=2)) + + +def failover(base_url, target_index): + containers = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], + cwd=ROOT, capture_output=True, text=True, check=True, + ).stdout.splitlines() + assert len(containers) == 2, "expected two running API replicas" + target = containers[target_index - 1] + target_name = subprocess.run( + ["docker", "inspect", "-f", "{{.Name}}", target], + capture_output=True, text=True, check=True, + ).stdout.strip().lstrip("/") + alias = subprocess.run( + ["docker", "exec", target, "python", "-c", + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])"], + capture_output=True, text=True, check=True, + ).stdout.strip() + client = Client(base_url, None, timeout=2) + alice, bob, _, chat = client.bootstrap() + started = time.monotonic() + stop = False + action_done = False + + def traffic(write): + local = Client(base_url, None, timeout=2) + confirmed, errors, after_stop = [], 0, None + key, body = uuid4().hex, "failover-" + uuid4().hex + while not stop: + try: + if write: + result = local.message(alice, chat, body, key=key) + confirmed.append(result["json"]) + key, body = uuid4().hex, "failover-" + uuid4().hex + else: + local.call("GET", "/api/v1/users/me", actor=alice, schema="User") + confirmed.append(1) + if action_done and after_stop is None: + after_stop = time.monotonic() - started + except Failure: + errors += 1 + time.sleep(0.05) + return confirmed, errors, after_stop + + with ThreadPoolExecutor(max_workers=2) as pool: + writes = pool.submit(traffic, True) + reads = pool.submit(traffic, False) + try: + proc = subprocess.run( + [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env={**os.environ, "TARGET_INSTANCE_ID": alias}, capture_output=True, text=True, timeout=20, + ) + action_done = True + stop_elapsed = time.monotonic() - started + assert proc.returncode == 0, "stop-one hook failed: " + proc.stderr.strip() + time.sleep(max(0, 15 - stop_elapsed)) + finally: + stop = True + confirmed, write_errors, first_write = writes.result() + read_count, read_errors, first_read = reads.result() + + actual = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Status}}", target], + capture_output=True, text=True, check=True, + ).stdout.strip() + assert actual == "exited", "target container remained running" + messages = history(client, bob, chat) + seen = {message["id"] for message in messages} + assert confirmed and read_count, "traffic did not continue" + assert {message["id"] for message in confirmed}.issubset(seen), "confirmed write lost" + assert first_write is not None and first_write <= 10, "write recovery exceeded 10 seconds" + assert first_read is not None and first_read <= 10, "read recovery exceeded 10 seconds" + elapsed = time.monotonic() - started + print(json.dumps({ + "action": "stop one selected API replica", "target_container": target_name, + "target_container_id": target, "target_state_after_hook": actual, + "elapsed_seconds": round(elapsed, 3), "stop_hook_seconds": round(stop_elapsed, 3), + "successful_reads": len(read_count), "failed_reads": read_errors, + "successful_writes": len(confirmed), "failed_write_attempts": write_errors, + "successful_rps": round((len(read_count) + len(confirmed)) / elapsed, 3), + "first_read_after_action_seconds": round(first_read, 3), + "first_write_after_action_seconds": round(first_write, 3), + "confirmed_messages_read_back": True, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--rolling", action="store_true") + parser.add_argument("--failover-index", type=int, choices=(1, 2)) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + args, remaining = parser.parse_known_args() + if args.rolling: + rolling(args.base_url) + elif args.failover_index: + failover(args.base_url, args.failover_index) + else: + unittest.main(argv=[sys.argv[0], *remaining]) -- 2.54.0 From 750e56d0c61fcb8ef602661006f256423a6e6589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:59:18 +0300 Subject: [PATCH 7/9] Measure scaling cost and confirm two-replica operating point --- REPORT.md | 38 ++- evidence/scale/baseline.jsonl | 54 ++++ evidence/scale/confirmation-4-to-2.jsonl | 6 + evidence/scale/fixed-p95.svg | 33 +++ evidence/scale/fixed-rps.svg | 33 +++ evidence/scale/growth-p95.svg | 33 +++ evidence/scale/growth-rps.svg | 33 +++ evidence/scale/summary.csv | 19 ++ scripts/benchmark.py | 322 +++++++++++++++++++++++ scripts/benchmark.sh | 4 + scripts/plot_scale.py | 86 ++++++ src/api.py | 11 +- student-tests/test_benchmark_profile.py | 72 +++++ 13 files changed, 739 insertions(+), 5 deletions(-) create mode 100644 evidence/scale/baseline.jsonl create mode 100644 evidence/scale/confirmation-4-to-2.jsonl create mode 100644 evidence/scale/fixed-p95.svg create mode 100644 evidence/scale/fixed-rps.svg create mode 100644 evidence/scale/growth-p95.svg create mode 100644 evidence/scale/growth-rps.svg create mode 100644 evidence/scale/summary.csv create mode 100644 scripts/benchmark.py create mode 100755 scripts/benchmark.sh create mode 100644 scripts/plot_scale.py create mode 100644 student-tests/test_benchmark_profile.py diff --git a/REPORT.md b/REPORT.md index 55d92527..8e17dadb 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 3 - Автор: Самохвалов Андрей Ремович, БПИ-2302. -- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 проверен ниже; исследование трека 1 выполняется отдельно. +- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 и выбранный трек проверены ниже. - Ветка: `solution/bpi2302-samohvalov`; commit/PR после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/`, публичные тесты репозитория, [HAProxy 3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). Код и текст подготовлены с помощью Codex; результаты ниже получены реальным запуском. @@ -81,6 +81,38 @@ python3 student-tests/test_operations.py --rolling --base-url http://127.0.0.1:8 Это **поочерёдная замена той же версии образа**: ни совместимость двух разных бинарных версий, ни миграция схемы во время нагрузки не проверялись. `migrate` выполняется до старта API, а скрипт замены запускает только API без повторной миграции. Для будущей смены схемы нужна отдельная совместимая миграция; данный опыт её не доказывает. -## Ограничения и дальнейшие измерения +## Уровень 5: цена масштабирования -Стенд имеет один Docker-host, один LB и один PostgreSQL: отказ узла или БД не переживает. `depends_on` гарантирует порядок запуска, а маршрутизацию выполняет HAProxy. Сценарии U4 показывают работоспособность и ошибки во время замены, но не предельную RPS при 1/2/4 репликах; отдельный эксперимент трека 1 будет добавлен для уровня 5. Генератор `tests/load.py` замкнутый и не устраняет coordinated omission. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. +**Гипотеза.** При фиксированном суммарном лимите API рост с двух до четырёх процессов не даст линейного выигрыша: каждому останется только 0.25 CPU, а конкуренция за пул и планировщик увеличит задержку. Если процессам разрешить суммарно больше CPU, пропускная способность вырастет, но на этой машине рост должен замедлиться около её двух виртуальных CPU. + +**Профиль и воспроизведение.** Измерения выполнены 2026-10-02 на том же macOS arm64 / Colima Docker-host с 2 CPU и 2.05 GB RAM, Docker Engine 29.5.2, Compose 5.3.1, образами и Python из раздела запуска. PostgreSQL оставался при 1 CPU/512 MiB, HAProxy при 0.5 CPU/128 MiB; pool каждой реплики ограничен восемью соединениями. Использован один синтетический чат с 1000 заранее записанными сообщениями; общий счётчик БД во время матрицы оставался 30969. Само измерение только читает последнюю страницу `/messages?limit=20` через LB с действующим Bearer. Файл `secrets/scale_fixture.json` хранит локальный тестовый токен и не включён в Git. Набор сообщений и запрос не менялись между точками и контрольным сравнением. + +`scripts/benchmark.sh` использует Python stdlib и задаёт *открытый* график поступления GET по монотонным часам: 64 потока, очередь максимум 256, HTTP keep-alive, timeout 4 с. У каждой точки 2 с прогрева и три независимых повтора по 8 с; заданная интенсивность 250/750/1500 запросов/с. Задержка успеха считается от запланированного момента поступления до полного ответа, включая ожидание в очереди; p50/p95/p99 относятся только к успешным ответам. Неуспешные поступления учитывают как HTTP/transport ошибки, так и переполнение очереди отдельно. Во всей матрице `errors_by_type` пуст, поэтому все потери на 1500 запросах/с — **отброшенные генератором поступления**, а не ответы API 5xx. Это доля необслуженных поступлений при заданном ограничении очереди, не точный максимум сервера. Показатель CPU самого генератора не превысил 0.115 с процессорного времени на 1 с наблюдения. + +Для чистого clone после настройки `.env` и secrets из раздела запуска (по умолчанию порт 8080): + +```sh +./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --rates 250,750,1500 --duration 8 --warmup 2 --output evidence/scale/baseline.jsonl +python3 scripts/plot_scale.py evidence/scale/baseline.jsonl +./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --modes fixed --replicas 4,2 --rates 1500 --duration 8 --warmup 2 --output evidence/scale/confirmation-4-to-2.jsonl +docker compose --env-file .env down +``` + +Скрипт отказывается дописывать существующий output: для нового опыта укажите новое имя файла. Он сам поднимает Compose с временным override лимитов и меняет число реплик. В локальном опыте использован порт 8083, так как 8080 уже занят другим учебным стендом. Матрица `fixed` делила общий лимит API 1 CPU/512 MiB между 1/2/4 репликами (соответственно 1/512, 0.5/256, 0.25/128 на процесс). Серия `growth` оставляла каждому процессу 0.5 CPU/256 MiB, поэтому сумма росла от 0.5/256 до 2 CPU/1024 MiB. Docker-host при этом **не** получил дополнительных физических ресурсов; 2 CPU у четырёх API конкурировали с БД и LB в VM на 2 CPU. Конфигурация override и фактические квоты зафиксированы в каждой JSONL-строке. + +Полные 54 результата с индивидуальными CPU/RAM API, PostgreSQL и LB, PostgreSQL counters/размером, pool waits, длительностью, RPS, ошибками и p50/p95/p99: [`evidence/scale/baseline.jsonl`](evidence/scale/baseline.jsonl). Медианы трёх повторов всех 18 точек: [`summary.csv`](evidence/scale/summary.csv). Графики медианных RPS и p95: [`fixed-rps.svg`](evidence/scale/fixed-rps.svg), [`fixed-p95.svg`](evidence/scale/fixed-p95.svg), [`growth-rps.svg`](evidence/scale/growth-rps.svg), [`growth-p95.svg`](evidence/scale/growth-p95.svg). + +| Серия, API | Успешные RPS при 750 | Успешные RPS при 1500 | Необслуженные при 1500 | p50 / p95 / p99 при 1500, мс | API / DB / LB CPU при 1500 | +| --- | ---: | ---: | ---: | ---: | ---: | +| fixed, 1 | 747 | 780 | 42.85% | 379 / 467 / 725 | 96 / 13 / 4% | +| fixed, 2 | 750 | 957 | 33.88% | 315 / 468 / 516 | 99 / 14 / 5% | +| fixed, 4 | 733 | 694 | 50.82% | 414 / 748 / 851 | 87 / 14 / 4% | +| growth, 1 | 362 | 385 | 72.09% | 797 / 1018 / 1070 | 51 / 6 / 2% | +| growth, 2 | 750 | 938 | 35.12% | 321 / 494 / 548 | 98 / 14 / 5% | +| growth, 4 | 747 | 1128 | 20.58% | 231 / 457 / 547 | 143 / 20 / 7% | + +**Насыщение и улучшение.** При суммарном 1 CPU рост заданной интенсивности с 750 до 1500 почти не увеличивает успешный RPS у одной реплики, а у четырёх уменьшает его; p95 и отброшенные поступления резко растут. API-группа использует около 87–99% одного CPU, PostgreSQL 13–14%, LB 4–5%; в этом профиле узкое место — CPU квота API и сопутствующее ожидание пула, **не** достигнутый предел CPU БД или LB. Например, `pool_wait_ms` у fixed/1 вырос с 51465 при 750 до 182073 при 1500; это суммарное время ожидания всех запросов за окно, не длительность отдельного запроса. Добавление ресурсов до growth/4 увеличило успешный RPS, но не линейно: API заняли 143% CPU при двух доступных VM CPU, DB 20%, LB 7%, а очередь всё ещё отбрасывала 20.58% поступлений. + +Измеримое эксплуатационное изменение — **выбрать 2 API вместо 4 при том же общем 1 CPU/512 MiB**. Отдельный контроль после возврата к неизменённому коду и тому же чату, 3 повтора сначала с 4, затем с 2 репликами по 8 с при 1500 заданных GET/с: [`confirmation-4-to-2.jsonl`](evidence/scale/confirmation-4-to-2.jsonl). В целевом чате по-прежнему 1000 сообщений; общий счётчик БД стал 30970 из-за одного независимого синтетического сообщения в проверке ACL между сериями. Медиана успешного RPS 856.033 → 952.317 (**+11.2%**), необслуженная доля 39.18% → 33.94% (−5.24 п.п.), p95 605.467 → 489.999 мс (−19.1%). Все три RPS с 2 API (940.679–972.715) выше всех трёх с 4 API (805.135–895.216). В `compose.yaml` по умолчанию уже стоят 2 API; для применения настройки после опыта с четырьмя: `docker compose --env-file .env up -d --wait --scale api=2`. Проверка публичного API после контроля — `make test BASE_URL=http://127.0.0.1:8083`, PASS. Это настройка топологии, а не ускорение SQL или исходного кода. + +**Пределы вывода.** Открытый график устраняет coordinated omission замкнутого `tests/load.py`, но конечные 64 потока, очередь 256 и локальный клиент влияют на измеренную границу. Отброшенные задания не дошли до HAProxy, а HTTP 5xx в серии не наблюдались. Каждый результат описывает этот GET-профиль на одном Docker-host, не запись сообщений и не универсальную ёмкость API. Кратковременные эффекты CPU throttling и фоновая нагрузка объясняют разброс между первой матрицей и контрольной серией; поэтому эффект выбора 2 API основан на отдельном сравнении с неперекрывающимися тремя повторами. Стенд имеет один LB и один PostgreSQL, не переживает потерю хоста/БД. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. diff --git a/evidence/scale/baseline.jsonl b/evidence/scale/baseline.jsonl new file mode 100644 index 00000000..f44615e9 --- /dev/null +++ b/evidence/scale/baseline.jsonl @@ -0,0 +1,54 @@ +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.95, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.831, "p95": 37.915, "p99": 211.151}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.384, "docker": {"api": {"avg_cpu_percent": 39.223, "max_memory_mib": 46.79}, "postgres": {"avg_cpu_percent": 6.993, "max_memory_mib": 53.62}, "lb": {"avg_cpu_percent": 2.253, "max_memory_mib": 41.58}}, "pool": {"observed_replicas": 1, "requests_queued": 75, "wait_ms": 600, "max_size": 8}, "postgres_delta": {"commits": 4018, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 65527, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.951, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.787, "p95": 3.429, "p99": 5.526}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.355, "docker": {"api": {"avg_cpu_percent": 26.653, "max_memory_mib": 47.33}, "postgres": {"avg_cpu_percent": 5.803, "max_memory_mib": 49.23}, "lb": {"avg_cpu_percent": 1.86, "max_memory_mib": 41.47}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4016, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61557, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.772, "p95": 3.703, "p99": 5.882}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.353, "docker": {"api": {"avg_cpu_percent": 29.56, "max_memory_mib": 45.91}, "postgres": {"avg_cpu_percent": 5.72, "max_memory_mib": 49.26}, "lb": {"avg_cpu_percent": 1.75, "max_memory_mib": 41.62}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4022, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61093, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.986, "elapsed_seconds": 8.032, "successful_latency_ms": {"p50": 4.909, "p95": 121.369, "p99": 137.788}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.707, "docker": {"api": {"avg_cpu_percent": 88.997, "max_memory_mib": 51.91}, "postgres": {"avg_cpu_percent": 12.417, "max_memory_mib": 49.31}, "lb": {"avg_cpu_percent": 4.683, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 4570, "wait_ms": 55096, "max_size": 8}, "postgres_delta": {"commits": 12605, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 191767, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.783, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.679, "p95": 85.479, "p99": 125.898}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.636, "docker": {"api": {"avg_cpu_percent": 60.08, "max_memory_mib": 52.52}, "postgres": {"avg_cpu_percent": 10.823, "max_memory_mib": 49.39}, "lb": {"avg_cpu_percent": 3.9, "max_memory_mib": 41.6}}, "pool": {"observed_replicas": 1, "requests_queued": 1631, "wait_ms": 17243, "max_size": 8}, "postgres_delta": {"commits": 11926, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181241, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.631, "elapsed_seconds": 8.036, "successful_latency_ms": {"p50": 3.533, "p95": 103.334, "p99": 138.015}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.695, "docker": {"api": {"avg_cpu_percent": 84.993, "max_memory_mib": 52.01}, "postgres": {"avg_cpu_percent": 12.933, "max_memory_mib": 49.41}, "lb": {"avg_cpu_percent": 4.407, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 4243, "wait_ms": 51465, "max_size": 8}, "postgres_delta": {"commits": 11896, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178481, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6890, "failed_requests": 5110, "error_ratio": 0.425833, "successful_rps": 823.026, "elapsed_seconds": 8.372, "successful_latency_ms": {"p50": 378.91, "p95": 447.591, "p99": 470.365}, "errors_by_type": {}, "queue_dropped": 5110, "client_cpu_seconds": 0.72, "docker": {"api": {"avg_cpu_percent": 97.463, "max_memory_mib": 64.79}, "postgres": {"avg_cpu_percent": 12.573, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 4.42, "max_memory_mib": 41.61}}, "pool": {"observed_replicas": 1, "requests_queued": 12335, "wait_ms": 188633, "max_size": 8}, "postgres_delta": {"commits": 14324, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 215405, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6858, "failed_requests": 5142, "error_ratio": 0.4285, "successful_rps": 780.299, "elapsed_seconds": 8.789, "successful_latency_ms": {"p50": 372.063, "p95": 466.704, "p99": 1071.218}, "errors_by_type": {}, "queue_dropped": 5142, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 96.24, "max_memory_mib": 53.53}, "postgres": {"avg_cpu_percent": 13.42, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 4.423, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 11597, "wait_ms": 182073, "max_size": 8}, "postgres_delta": {"commits": 14394, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216827, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6445, "failed_requests": 5555, "error_ratio": 0.462917, "successful_rps": 766.917, "elapsed_seconds": 8.404, "successful_latency_ms": {"p50": 382.659, "p95": 600.998, "p99": 725.297}, "errors_by_type": {}, "queue_dropped": 5555, "client_cpu_seconds": 0.75, "docker": {"api": {"avg_cpu_percent": 96.137, "max_memory_mib": 53.47}, "postgres": {"avg_cpu_percent": 15.257, "max_memory_mib": 49.48}, "lb": {"avg_cpu_percent": 4.627, "max_memory_mib": 41.67}}, "pool": {"observed_replicas": 1, "requests_queued": 11447, "wait_ms": 180076, "max_size": 8}, "postgres_delta": {"commits": 13337, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 200363, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.96, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.769, "p95": 13.026, "p99": 51.133}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.342, "docker": {"api": {"avg_cpu_percent": 38.633, "max_memory_mib": 85.02}, "postgres": {"avg_cpu_percent": 5.927, "max_memory_mib": 62.68}, "lb": {"avg_cpu_percent": 1.71, "max_memory_mib": 41.82}}, "pool": {"observed_replicas": 2, "requests_queued": 5, "wait_ms": 4, "max_size": 8}, "postgres_delta": {"commits": 4198, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71471, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.959, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.875, "p95": 33.197, "p99": 100.083}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.356, "docker": {"api": {"avg_cpu_percent": 43.583, "max_memory_mib": 112.32}, "postgres": {"avg_cpu_percent": 5.957, "max_memory_mib": 64.63}, "lb": {"avg_cpu_percent": 1.9, "max_memory_mib": 41.56}}, "pool": {"observed_replicas": 2, "requests_queued": 3, "wait_ms": 14, "max_size": 8}, "postgres_delta": {"commits": 4072, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61681, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.937, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.873, "p95": 25.949, "p99": 138.238}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.372, "docker": {"api": {"avg_cpu_percent": 42.983, "max_memory_mib": 102.69}, "postgres": {"avg_cpu_percent": 6.063, "max_memory_mib": 64.66}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 2, "requests_queued": 31, "wait_ms": 78, "max_size": 8}, "postgres_delta": {"commits": 3777, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 57121, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.804, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.755, "p95": 108.359, "p99": 175.477}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.623, "docker": {"api": {"avg_cpu_percent": 69.88, "max_memory_mib": 108.39}, "postgres": {"avg_cpu_percent": 12.08, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 3.88, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1294, "wait_ms": 7867, "max_size": 8}, "postgres_delta": {"commits": 12232, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184771, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.772, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.731, "p95": 100.322, "p99": 162.856}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.61, "docker": {"api": {"avg_cpu_percent": 70.053, "max_memory_mib": 96.87}, "postgres": {"avg_cpu_percent": 12.173, "max_memory_mib": 65.0}, "lb": {"avg_cpu_percent": 3.93, "max_memory_mib": 41.97}}, "pool": {"observed_replicas": 2, "requests_queued": 1118, "wait_ms": 5573, "max_size": 8}, "postgres_delta": {"commits": 12156, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183345, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.734, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 1.816, "p95": 178.504, "p99": 224.107}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.627, "docker": {"api": {"avg_cpu_percent": 68.573, "max_memory_mib": 97.41}, "postgres": {"avg_cpu_percent": 12.37, "max_memory_mib": 64.76}, "lb": {"avg_cpu_percent": 3.767, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1324, "wait_ms": 11493, "max_size": 8}, "postgres_delta": {"commits": 12142, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183157, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7863, "failed_requests": 4137, "error_ratio": 0.34475, "successful_rps": 941.177, "elapsed_seconds": 8.354, "successful_latency_ms": {"p50": 319.914, "p95": 473.58, "p99": 530.577}, "errors_by_type": {}, "queue_dropped": 4137, "client_cpu_seconds": 0.785, "docker": {"api": {"avg_cpu_percent": 98.37, "max_memory_mib": 109.51}, "postgres": {"avg_cpu_percent": 14.65, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 5.09, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 6822, "wait_ms": 77320, "max_size": 8}, "postgres_delta": {"commits": 14978, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 225169, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8293, "failed_requests": 3707, "error_ratio": 0.308917, "successful_rps": 993.827, "elapsed_seconds": 8.345, "successful_latency_ms": {"p50": 297.841, "p95": 454.544, "p99": 505.25}, "errors_by_type": {}, "queue_dropped": 3707, "client_cpu_seconds": 0.762, "docker": {"api": {"avg_cpu_percent": 100.243, "max_memory_mib": 113.26}, "postgres": {"avg_cpu_percent": 14.147, "max_memory_mib": 65.02}, "lb": {"avg_cpu_percent": 4.87, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 5427, "wait_ms": 49740, "max_size": 8}, "postgres_delta": {"commits": 15964, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 240127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7934, "failed_requests": 4066, "error_ratio": 0.338833, "successful_rps": 956.631, "elapsed_seconds": 8.294, "successful_latency_ms": {"p50": 314.99, "p95": 467.748, "p99": 516.41}, "errors_by_type": {}, "queue_dropped": 4066, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 98.82, "max_memory_mib": 122.91}, "postgres": {"avg_cpu_percent": 13.203, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 4.807, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 2, "requests_queued": 5707, "wait_ms": 72376, "max_size": 8}, "postgres_delta": {"commits": 16857, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253657, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.958, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.016, "p95": 61.385, "p99": 81.718}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.313, "docker": {"api": {"avg_cpu_percent": 64.537, "max_memory_mib": 205.98}, "postgres": {"avg_cpu_percent": 5.16, "max_memory_mib": 79.56}, "lb": {"avg_cpu_percent": 1.72, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 4, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4223, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 76658, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.944, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.944, "p95": 59.768, "p99": 80.401}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.323, "docker": {"api": {"avg_cpu_percent": 60.467, "max_memory_mib": 205.43}, "postgres": {"avg_cpu_percent": 5.073, "max_memory_mib": 79.71}, "lb": {"avg_cpu_percent": 1.657, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4020, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60455, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.021, "p95": 60.482, "p99": 79.776}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 51.723, "max_memory_mib": 203.98}, "postgres": {"avg_cpu_percent": 5.303, "max_memory_mib": 79.81}, "lb": {"avg_cpu_percent": 1.68, "max_memory_mib": 41.55}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4101, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61639, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 744.222, "elapsed_seconds": 8.062, "successful_latency_ms": {"p50": 3.319, "p95": 308.301, "p99": 415.861}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.613, "docker": {"api": {"avg_cpu_percent": 83.097, "max_memory_mib": 205.59}, "postgres": {"avg_cpu_percent": 13.187, "max_memory_mib": 95.02}, "lb": {"avg_cpu_percent": 3.927, "max_memory_mib": 42.05}}, "pool": {"observed_replicas": 4, "requests_queued": 952, "wait_ms": 12771, "max_size": 8}, "postgres_delta": {"commits": 11710, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181824, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 713.504, "elapsed_seconds": 8.409, "successful_latency_ms": {"p50": 126.994, "p95": 414.136, "p99": 480.195}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.582, "docker": {"api": {"avg_cpu_percent": 92.41, "max_memory_mib": 207.79}, "postgres": {"avg_cpu_percent": 12.73, "max_memory_mib": 95.05}, "lb": {"avg_cpu_percent": 3.723, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 4, "requests_queued": 2167, "wait_ms": 26550, "max_size": 8}, "postgres_delta": {"commits": 12191, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183838, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 732.997, "elapsed_seconds": 8.186, "successful_latency_ms": {"p50": 110.445, "p95": 397.681, "p99": 486.245}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.585, "docker": {"api": {"avg_cpu_percent": 93.953, "max_memory_mib": 217.25}, "postgres": {"avg_cpu_percent": 12.887, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.673, "max_memory_mib": 42.28}}, "pool": {"observed_replicas": 4, "requests_queued": 1901, "wait_ms": 24258, "max_size": 8}, "postgres_delta": {"commits": 11844, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178211, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6950, "failed_requests": 5050, "error_ratio": 0.420833, "successful_rps": 811.023, "elapsed_seconds": 8.569, "successful_latency_ms": {"p50": 315.472, "p95": 633.293, "p99": 771.213}, "errors_by_type": {}, "queue_dropped": 5050, "client_cpu_seconds": 0.698, "docker": {"api": {"avg_cpu_percent": 98.617, "max_memory_mib": 216.33}, "postgres": {"avg_cpu_percent": 15.563, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 4.53, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 3048, "wait_ms": 32524, "max_size": 8}, "postgres_delta": {"commits": 14083, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211553, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5902, "failed_requests": 6098, "error_ratio": 0.508167, "successful_rps": 693.857, "elapsed_seconds": 8.506, "successful_latency_ms": {"p50": 413.811, "p95": 769.612, "p99": 855.228}, "errors_by_type": {}, "queue_dropped": 6098, "client_cpu_seconds": 0.674, "docker": {"api": {"avg_cpu_percent": 80.183, "max_memory_mib": 204.21}, "postgres": {"avg_cpu_percent": 11.61, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.993, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 2088, "wait_ms": 33271, "max_size": 8}, "postgres_delta": {"commits": 12393, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 186123, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5756, "failed_requests": 6244, "error_ratio": 0.520333, "successful_rps": 681.037, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 420.287, "p95": 747.7, "p99": 851.226}, "errors_by_type": {}, "queue_dropped": 6244, "client_cpu_seconds": 0.604, "docker": {"api": {"avg_cpu_percent": 87.06, "max_memory_mib": 194.64}, "postgres": {"avg_cpu_percent": 13.59, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.353, "max_memory_mib": 42.36}}, "pool": {"observed_replicas": 4, "requests_queued": 2203, "wait_ms": 38980, "max_size": 8}, "postgres_delta": {"commits": 11550, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 173489, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.946, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.572, "p95": 206.839, "p99": 277.387}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 33.947, "max_memory_mib": 48.01}, "postgres": {"avg_cpu_percent": 6.093, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 463, "wait_ms": 4901, "max_size": 8}, "postgres_delta": {"commits": 4114, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 66849, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.365, "p95": 39.928, "p99": 72.726}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.262, "docker": {"api": {"avg_cpu_percent": 28.06, "max_memory_mib": 48.11}, "postgres": {"avg_cpu_percent": 5.183, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 1.587, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 135, "wait_ms": 196, "max_size": 8}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61300, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.977, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.684, "p95": 59.465, "p99": 80.414}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.345, "docker": {"api": {"avg_cpu_percent": 34.943, "max_memory_mib": 47.95}, "postgres": {"avg_cpu_percent": 6.29, "max_memory_mib": 49.47}, "lb": {"avg_cpu_percent": 2.08, "max_memory_mib": 41.66}}, "pool": {"observed_replicas": 1, "requests_queued": 210, "wait_ms": 410, "max_size": 8}, "postgres_delta": {"commits": 3956, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60059, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3164, "failed_requests": 2836, "error_ratio": 0.472667, "successful_rps": 362.33, "elapsed_seconds": 8.732, "successful_latency_ms": {"p50": 823.553, "p95": 1071.351, "p99": 1182.579}, "errors_by_type": {}, "queue_dropped": 2836, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 50.06, "max_memory_mib": 64.7}, "postgres": {"avg_cpu_percent": 6.51, "max_memory_mib": 49.75}, "lb": {"avg_cpu_percent": 2.317, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5298, "wait_ms": 184862, "max_size": 8}, "postgres_delta": {"commits": 5828, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 88215, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3140, "failed_requests": 2860, "error_ratio": 0.476667, "successful_rps": 347.987, "elapsed_seconds": 9.023, "successful_latency_ms": {"p50": 809.855, "p95": 1190.388, "p99": 1238.353}, "errors_by_type": {}, "queue_dropped": 2860, "client_cpu_seconds": 0.393, "docker": {"api": {"avg_cpu_percent": 50.673, "max_memory_mib": 64.82}, "postgres": {"avg_cpu_percent": 7.463, "max_memory_mib": 50.64}, "lb": {"avg_cpu_percent": 2.083, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 5266, "wait_ms": 191351, "max_size": 8}, "postgres_delta": {"commits": 6465, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 97643, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3410, "failed_requests": 2590, "error_ratio": 0.431667, "successful_rps": 391.052, "elapsed_seconds": 8.72, "successful_latency_ms": {"p50": 795.636, "p95": 976.194, "p99": 988.076}, "errors_by_type": {}, "queue_dropped": 2590, "client_cpu_seconds": 0.38, "docker": {"api": {"avg_cpu_percent": 50.493, "max_memory_mib": 61.84}, "postgres": {"avg_cpu_percent": 6.57, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.033, "max_memory_mib": 41.93}}, "pool": {"observed_replicas": 1, "requests_queued": 5799, "wait_ms": 185658, "max_size": 8}, "postgres_delta": {"commits": 7011, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 105675, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3475, "failed_requests": 8525, "error_ratio": 0.710417, "successful_rps": 399.011, "elapsed_seconds": 8.709, "successful_latency_ms": {"p50": 795.926, "p95": 918.481, "p99": 980.371}, "errors_by_type": {}, "queue_dropped": 8525, "client_cpu_seconds": 0.419, "docker": {"api": {"avg_cpu_percent": 50.17, "max_memory_mib": 65.2}, "postgres": {"avg_cpu_percent": 6.02, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 1.923, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 1, "requests_queued": 6018, "wait_ms": 195259, "max_size": 8}, "postgres_delta": {"commits": 6592, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 99497, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3152, "failed_requests": 8848, "error_ratio": 0.737333, "successful_rps": 354.029, "elapsed_seconds": 8.903, "successful_latency_ms": {"p50": 878.869, "p95": 1018.139, "p99": 1070.497}, "errors_by_type": {}, "queue_dropped": 8848, "client_cpu_seconds": 0.44, "docker": {"api": {"avg_cpu_percent": 51.427, "max_memory_mib": 52.96}, "postgres": {"avg_cpu_percent": 6.83, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.257, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5362, "wait_ms": 190945, "max_size": 8}, "postgres_delta": {"commits": 6204, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 93635, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3349, "failed_requests": 8651, "error_ratio": 0.720917, "successful_rps": 384.74, "elapsed_seconds": 8.705, "successful_latency_ms": {"p50": 797.336, "p95": 1034.655, "p99": 1084.912}, "errors_by_type": {}, "queue_dropped": 8651, "client_cpu_seconds": 0.442, "docker": {"api": {"avg_cpu_percent": 50.96, "max_memory_mib": 64.4}, "postgres": {"avg_cpu_percent": 5.623, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.29, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 5539, "wait_ms": 185287, "max_size": 8}, "postgres_delta": {"commits": 6874, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 103703, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.98, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.87, "p95": 10.076, "p99": 50.451}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.319, "docker": {"api": {"avg_cpu_percent": 34.92, "max_memory_mib": 85.09}, "postgres": {"avg_cpu_percent": 6.023, "max_memory_mib": 58.95}, "lb": {"avg_cpu_percent": 1.613, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 6, "wait_ms": 1, "max_size": 7}, "postgres_delta": {"commits": 4102, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 67511, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.936, "p95": 55.026, "p99": 129.173}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 41.043, "max_memory_mib": 96.67}, "postgres": {"avg_cpu_percent": 8.053, "max_memory_mib": 64.64}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 2, "requests_queued": 15, "wait_ms": 308, "max_size": 8}, "postgres_delta": {"commits": 4081, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61880, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.982, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.886, "p95": 15.37, "p99": 54.353}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.341, "docker": {"api": {"avg_cpu_percent": 38.787, "max_memory_mib": 98.52}, "postgres": {"avg_cpu_percent": 5.4, "max_memory_mib": 64.68}, "lb": {"avg_cpu_percent": 1.707, "max_memory_mib": 41.59}}, "pool": {"observed_replicas": 2, "requests_queued": 1, "wait_ms": 2, "max_size": 8}, "postgres_delta": {"commits": 4079, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61581, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 5903, "failed_requests": 97, "error_ratio": 0.016167, "successful_rps": 728.191, "elapsed_seconds": 8.106, "successful_latency_ms": {"p50": 2.148, "p95": 468.108, "p99": 599.437}, "errors_by_type": {}, "queue_dropped": 97, "client_cpu_seconds": 0.647, "docker": {"api": {"avg_cpu_percent": 79.57, "max_memory_mib": 107.99}, "postgres": {"avg_cpu_percent": 13.603, "max_memory_mib": 64.71}, "lb": {"avg_cpu_percent": 4.013, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 1869, "wait_ms": 27621, "max_size": 8}, "postgres_delta": {"commits": 11975, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180645, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.669, "elapsed_seconds": 8.004, "successful_latency_ms": {"p50": 2.292, "p95": 144.278, "p99": 193.517}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.716, "docker": {"api": {"avg_cpu_percent": 92.29, "max_memory_mib": 108.7}, "postgres": {"avg_cpu_percent": 13.147, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.19, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 2115, "wait_ms": 15549, "max_size": 8}, "postgres_delta": {"commits": 12108, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.762, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.224, "p95": 167.623, "p99": 254.554}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.731, "docker": {"api": {"avg_cpu_percent": 86.027, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.06, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.49, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 1856, "wait_ms": 14945, "max_size": 8}, "postgres_delta": {"commits": 11997, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 179629, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7786, "failed_requests": 4214, "error_ratio": 0.351167, "successful_rps": 938.324, "elapsed_seconds": 8.298, "successful_latency_ms": {"p50": 321.188, "p95": 493.5, "p99": 548.275}, "errors_by_type": {}, "queue_dropped": 4214, "client_cpu_seconds": 0.739, "docker": {"api": {"avg_cpu_percent": 98.067, "max_memory_mib": 99.7}, "postgres": {"avg_cpu_percent": 14.437, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 4.373, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7974, "wait_ms": 76567, "max_size": 8}, "postgres_delta": {"commits": 15558, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 233671, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7921, "failed_requests": 4079, "error_ratio": 0.339917, "successful_rps": 953.293, "elapsed_seconds": 8.309, "successful_latency_ms": {"p50": 321.964, "p95": 454.678, "p99": 509.653}, "errors_by_type": {}, "queue_dropped": 4079, "client_cpu_seconds": 0.758, "docker": {"api": {"avg_cpu_percent": 100.757, "max_memory_mib": 100.03}, "postgres": {"avg_cpu_percent": 15.21, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.527, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 2, "requests_queued": 9150, "wait_ms": 89177, "max_size": 8}, "postgres_delta": {"commits": 16078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 241647, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7486, "failed_requests": 4514, "error_ratio": 0.376167, "successful_rps": 900.488, "elapsed_seconds": 8.313, "successful_latency_ms": {"p50": 321.365, "p95": 527.564, "p99": 574.568}, "errors_by_type": {}, "queue_dropped": 4514, "client_cpu_seconds": 0.76, "docker": {"api": {"avg_cpu_percent": 96.313, "max_memory_mib": 114.13}, "postgres": {"avg_cpu_percent": 13.923, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.857, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7416, "wait_ms": 85412, "max_size": 8}, "postgres_delta": {"commits": 15026, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 226057, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.9, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.847, "p95": 17.892, "p99": 116.852}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.335, "docker": {"api": {"avg_cpu_percent": 73.133, "max_memory_mib": 165.93}, "postgres": {"avg_cpu_percent": 5.717, "max_memory_mib": 70.38}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 4, "requests_queued": 45, "wait_ms": 237, "max_size": 5}, "postgres_delta": {"commits": 4090, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71220, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.719, "elapsed_seconds": 8.009, "successful_latency_ms": {"p50": 2.907, "p95": 9.687, "p99": 21.273}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.328, "docker": {"api": {"avg_cpu_percent": 49.73, "max_memory_mib": 191.97}, "postgres": {"avg_cpu_percent": 5.503, "max_memory_mib": 70.47}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 5}, "postgres_delta": {"commits": 4110, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61812, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.939, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 3.098, "p95": 17.675, "p99": 76.487}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.402, "docker": {"api": {"avg_cpu_percent": 74.683, "max_memory_mib": 177.49}, "postgres": {"avg_cpu_percent": 7.597, "max_memory_mib": 72.36}, "lb": {"avg_cpu_percent": 1.89, "max_memory_mib": 41.57}}, "pool": {"observed_replicas": 4, "requests_queued": 1, "wait_ms": 22, "max_size": 5}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 62186, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 747.495, "elapsed_seconds": 8.027, "successful_latency_ms": {"p50": 4.456, "p95": 312.888, "p99": 369.613}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.793, "docker": {"api": {"avg_cpu_percent": 109.91, "max_memory_mib": 213.65}, "postgres": {"avg_cpu_percent": 16.54, "max_memory_mib": 95.11}, "lb": {"avg_cpu_percent": 4.963, "max_memory_mib": 42.07}}, "pool": {"observed_replicas": 4, "requests_queued": 350, "wait_ms": 3249, "max_size": 8}, "postgres_delta": {"commits": 12366, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 193680, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 748.081, "elapsed_seconds": 8.021, "successful_latency_ms": {"p50": 2.363, "p95": 25.407, "p99": 53.156}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.616, "docker": {"api": {"avg_cpu_percent": 111.363, "max_memory_mib": 207.52}, "postgres": {"avg_cpu_percent": 13.387, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.747, "max_memory_mib": 41.89}}, "pool": {"observed_replicas": 4, "requests_queued": 23, "wait_ms": 57, "max_size": 8}, "postgres_delta": {"commits": 12044, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180901, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 741.745, "elapsed_seconds": 8.089, "successful_latency_ms": {"p50": 2.595, "p95": 182.257, "p99": 375.515}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.74, "docker": {"api": {"avg_cpu_percent": 117.407, "max_memory_mib": 220.91}, "postgres": {"avg_cpu_percent": 15.23, "max_memory_mib": 95.18}, "lb": {"avg_cpu_percent": 4.083, "max_memory_mib": 42.04}}, "pool": {"observed_replicas": 4, "requests_queued": 357, "wait_ms": 3544, "max_size": 8}, "postgres_delta": {"commits": 12159, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184013, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9320, "failed_requests": 2680, "error_ratio": 0.223333, "successful_rps": 1119.586, "elapsed_seconds": 8.325, "successful_latency_ms": {"p50": 230.705, "p95": 466.032, "p99": 733.365}, "errors_by_type": {}, "queue_dropped": 2680, "client_cpu_seconds": 0.938, "docker": {"api": {"avg_cpu_percent": 142.593, "max_memory_mib": 218.29}, "postgres": {"avg_cpu_percent": 20.28, "max_memory_mib": 96.34}, "lb": {"avg_cpu_percent": 6.71, "max_memory_mib": 42.43}}, "pool": {"observed_replicas": 4, "requests_queued": 1799, "wait_ms": 17211, "max_size": 8}, "postgres_delta": {"commits": 17785, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 267049, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9886, "failed_requests": 2114, "error_ratio": 0.176167, "successful_rps": 1170.613, "elapsed_seconds": 8.445, "successful_latency_ms": {"p50": 226.066, "p95": 413.16, "p99": 489.537}, "errors_by_type": {}, "queue_dropped": 2114, "client_cpu_seconds": 0.941, "docker": {"api": {"avg_cpu_percent": 142.217, "max_memory_mib": 221.61}, "postgres": {"avg_cpu_percent": 20.977, "max_memory_mib": 95.24}, "lb": {"avg_cpu_percent": 6.13, "max_memory_mib": 42.45}}, "pool": {"observed_replicas": 4, "requests_queued": 1972, "wait_ms": 27198, "max_size": 8}, "postgres_delta": {"commits": 20148, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 302329, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9531, "failed_requests": 2469, "error_ratio": 0.20575, "successful_rps": 1127.648, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 246.905, "p95": 456.887, "p99": 547.176}, "errors_by_type": {}, "queue_dropped": 2469, "client_cpu_seconds": 0.97, "docker": {"api": {"avg_cpu_percent": 144.297, "max_memory_mib": 235.65}, "postgres": {"avg_cpu_percent": 19.94, "max_memory_mib": 97.48}, "lb": {"avg_cpu_percent": 6.687, "max_memory_mib": 42.46}}, "pool": {"observed_replicas": 4, "requests_queued": 1540, "wait_ms": 18841, "max_size": 8}, "postgres_delta": {"commits": 19840, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 297789, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} diff --git a/evidence/scale/confirmation-4-to-2.jsonl b/evidence/scale/confirmation-4-to-2.jsonl new file mode 100644 index 00000000..9bf14525 --- /dev/null +++ b/evidence/scale/confirmation-4-to-2.jsonl @@ -0,0 +1,6 @@ +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7298, "failed_requests": 4702, "error_ratio": 0.391833, "successful_rps": 856.033, "elapsed_seconds": 8.525, "successful_latency_ms": {"p50": 315.394, "p95": 605.467, "p99": 723.579}, "errors_by_type": {}, "queue_dropped": 4702, "client_cpu_seconds": 0.699, "docker": {"api": {"avg_cpu_percent": 101.373, "max_memory_mib": 230.97}, "postgres": {"avg_cpu_percent": 10.917, "max_memory_mib": 96.28}, "lb": {"avg_cpu_percent": 4.033, "max_memory_mib": 42.7}}, "pool": {"observed_replicas": 4, "requests_queued": 2961, "wait_ms": 49012, "max_size": 8}, "postgres_delta": {"commits": 14598, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211976, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7569, "failed_requests": 4431, "error_ratio": 0.36925, "successful_rps": 895.216, "elapsed_seconds": 8.455, "successful_latency_ms": {"p50": 302.528, "p95": 551.032, "p99": 821.366}, "errors_by_type": {}, "queue_dropped": 4431, "client_cpu_seconds": 0.705, "docker": {"api": {"avg_cpu_percent": 99.517, "max_memory_mib": 205.15}, "postgres": {"avg_cpu_percent": 15.823, "max_memory_mib": 96.31}, "lb": {"avg_cpu_percent": 4.787, "max_memory_mib": 42.63}}, "pool": {"observed_replicas": 4, "requests_queued": 4193, "wait_ms": 33869, "max_size": 8}, "postgres_delta": {"commits": 14895, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216728, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6745, "failed_requests": 5255, "error_ratio": 0.437917, "successful_rps": 805.135, "elapsed_seconds": 8.377, "successful_latency_ms": {"p50": 335.741, "p95": 691.364, "p99": 835.845}, "errors_by_type": {}, "queue_dropped": 5255, "client_cpu_seconds": 0.704, "docker": {"api": {"avg_cpu_percent": 99.893, "max_memory_mib": 202.03}, "postgres": {"avg_cpu_percent": 15.907, "max_memory_mib": 96.36}, "lb": {"avg_cpu_percent": 4.783, "max_memory_mib": 42.64}}, "pool": {"observed_replicas": 4, "requests_queued": 3062, "wait_ms": 48677, "max_size": 8}, "postgres_delta": {"commits": 13421, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 194846, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7927, "failed_requests": 4073, "error_ratio": 0.339417, "successful_rps": 952.317, "elapsed_seconds": 8.324, "successful_latency_ms": {"p50": 316.048, "p95": 495.87, "p99": 540.128}, "errors_by_type": {}, "queue_dropped": 4073, "client_cpu_seconds": 0.749, "docker": {"api": {"avg_cpu_percent": 96.897, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.917, "max_memory_mib": 66.3}, "lb": {"avg_cpu_percent": 3.933, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 6441, "wait_ms": 59041, "max_size": 8}, "postgres_delta": {"commits": 15327, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222787, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8055, "failed_requests": 3945, "error_ratio": 0.32875, "successful_rps": 972.715, "elapsed_seconds": 8.281, "successful_latency_ms": {"p50": 309.538, "p95": 465.748, "p99": 575.029}, "errors_by_type": {}, "queue_dropped": 3945, "client_cpu_seconds": 0.738, "docker": {"api": {"avg_cpu_percent": 94.727, "max_memory_mib": 100.38}, "postgres": {"avg_cpu_percent": 15.02, "max_memory_mib": 65.29}, "lb": {"avg_cpu_percent": 4.33, "max_memory_mib": 42.54}}, "pool": {"observed_replicas": 2, "requests_queued": 5546, "wait_ms": 59665, "max_size": 8}, "postgres_delta": {"commits": 17391, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253125, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7862, "failed_requests": 4138, "error_ratio": 0.344833, "successful_rps": 940.679, "elapsed_seconds": 8.358, "successful_latency_ms": {"p50": 319.584, "p95": 489.999, "p99": 595.768}, "errors_by_type": {}, "queue_dropped": 4138, "client_cpu_seconds": 0.725, "docker": {"api": {"avg_cpu_percent": 99.007, "max_memory_mib": 113.54}, "postgres": {"avg_cpu_percent": 15.377, "max_memory_mib": 68.17}, "lb": {"avg_cpu_percent": 4.427, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 7232, "wait_ms": 85079, "max_size": 8}, "postgres_delta": {"commits": 15296, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222495, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} diff --git a/evidence/scale/fixed-p95.svg b/evidence/scale/fixed-p95.svg new file mode 100644 index 00000000..2b887a48 --- /dev/null +++ b/evidence/scale/fixed-p95.svg @@ -0,0 +1,33 @@ + + +fixed: p95, мс + +0 + +206 + +411 + +617 + +822 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/fixed-rps.svg b/evidence/scale/fixed-rps.svg new file mode 100644 index 00000000..98ab80b2 --- /dev/null +++ b/evidence/scale/fixed-rps.svg @@ -0,0 +1,33 @@ + + +fixed: Успешные запросы/с + +0 + +263 + +526 + +789 + +1052 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/growth-p95.svg b/evidence/scale/growth-p95.svg new file mode 100644 index 00000000..0d9e327a --- /dev/null +++ b/evidence/scale/growth-p95.svg @@ -0,0 +1,33 @@ + + +growth: p95, мс + +0 + +295 + +589 + +884 + +1178 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/growth-rps.svg b/evidence/scale/growth-rps.svg new file mode 100644 index 00000000..153d0b28 --- /dev/null +++ b/evidence/scale/growth-rps.svg @@ -0,0 +1,33 @@ + + +growth: Успешные запросы/с + +0 + +310 + +620 + +930 + +1240 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/summary.csv b/evidence/scale/summary.csv new file mode 100644 index 00000000..9209371f --- /dev/null +++ b/evidence/scale/summary.csv @@ -0,0 +1,19 @@ +mode,replicas,rate,successful_rps,error_ratio,p50_ms,p95_ms,p99_ms,api_cpu_percent,postgres_cpu_percent,lb_cpu_percent,api_memory_mib,postgres_memory_mib,lb_memory_mib,pool_wait_ms,queue_dropped +fixed,1,250,249.951,0.0,2.787,3.703,5.882,29.56,5.803,1.86,46.79,49.26,41.58,0,0 +fixed,1,750,746.986,0.0,3.533,103.334,137.788,84.993,12.417,4.407,52.01,49.39,41.64,51465,0 +fixed,1,1500,780.299,0.4285,378.91,466.704,725.297,96.24,13.42,4.423,53.53,49.48,41.64,182073,5142 +fixed,2,250,249.959,0.0,2.873,25.949,100.083,42.983,5.957,1.753,102.69,64.63,41.64,14,0 +fixed,2,750,749.772,0.0,1.755,108.359,175.477,69.88,12.173,3.88,97.41,64.76,41.91,7867,0 +fixed,2,1500,956.631,0.338833,314.99,467.748,516.41,98.82,14.147,4.87,113.26,64.77,42.0,72376,4066 +fixed,4,250,249.958,0.0,3.016,60.482,80.401,60.467,5.16,1.68,205.43,79.71,41.74,0,0 +fixed,4,750,732.997,0.0,110.445,397.681,480.195,92.41,12.887,3.723,207.79,95.05,42.23,24258,0 +fixed,4,1500,693.857,0.508167,413.811,747.7,851.226,87.06,13.59,3.993,204.21,95.16,42.35,33271,6098 +growth,1,250,249.97,0.0,2.572,59.465,80.414,33.947,6.093,1.697,48.01,49.47,41.68,410,0 +growth,1,750,362.33,0.472667,809.855,1071.351,1182.579,50.493,6.57,2.083,64.7,49.75,41.69,185658,2836 +growth,1,1500,384.74,0.720917,797.336,1018.139,1070.497,50.96,6.02,2.257,64.4,49.52,41.7,190945,8651 +growth,2,250,249.98,0.0,2.886,15.37,54.353,38.787,6.023,1.707,96.67,64.64,41.7,2,0 +growth,2,750,749.669,0.0,2.224,167.623,254.554,86.027,13.147,4.19,107.99,64.73,41.94,15549,0 +growth,2,1500,938.324,0.351167,321.365,493.5,548.275,98.067,14.437,4.527,100.03,64.75,41.98,85412,4214 +growth,4,250,249.9,0.0,2.907,17.675,76.487,73.133,5.717,1.753,177.49,70.47,41.91,22,0 +growth,4,750,747.495,0.0,2.595,182.257,369.613,111.363,15.23,4.083,213.65,95.12,42.04,3249,0 +growth,4,1500,1127.648,0.20575,230.705,456.887,547.176,142.593,20.28,6.687,221.61,96.34,42.45,18841,2469 diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 00000000..cdcdf2eb --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Reproducible, rate-controlled GET benchmark for the lab3 scaling track.""" + +import argparse +from collections import defaultdict +import http.client +import json +import math +import os +from pathlib import Path +from queue import Full, Queue +import statistics +import subprocess +import sys +import tempfile +import threading +import time +from urllib.parse import urlsplit +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 - the repository's public bootstrap client + + +def topology(mode, replicas): + if mode not in ("fixed", "growth") or replicas not in (1, 2, 4): + raise ValueError("mode must be fixed/growth and replicas must be 1/2/4") + return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256) + + +def validate_matrix(rows): + if not rows: + raise ValueError("no benchmark rows") + rates = {row["rate"] for row in rows} + if len(rates) < 3: + raise ValueError("need at least three offered rates") + profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows} + if len(profiles) != 1: + raise ValueError("workload profile changed across measurements") + for mode in ("fixed", "growth"): + for replicas in (1, 2, 4): + _, cpu, memory = topology(mode, replicas) + for rate in rates: + repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)] + if sorted(repeats) != [1, 2, 3]: + raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}") + for row in rows: + if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory): + raise ValueError("incorrect API resource quota") + + +def percentile(values, fraction): + if not values: + return None + ordered = sorted(values) + return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3) + + +def command(*args): + return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip() + + +def compose(*args, override=None): + prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"] + if override: + prefix += ["-f", str(override)] + return command(*prefix, *args) + + +def pool_snapshot(base_url, replicas): + found = {} + for _ in range(8 * replicas): + with urlopen(base_url + "/health/ready", timeout=4) as response: + headers = response.headers + found[headers["X-Instance-Id"]] = { + "requests": int(headers["X-Pool-Requests"]), + "queued": int(headers["X-Pool-Queued"]), + "wait_ms": int(headers["X-Pool-Wait-Ms"]), + "size": int(headers["X-Pool-Size"]), + "available": int(headers["X-Pool-Available"]), + } + if len(found) == replicas: + break + return found + + +def db_snapshot(override): + sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback," + "blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) " + "FROM pg_stat_database WHERE datname=current_database()") + values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override), + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) + return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"), + map(int, values.split("|")))) + + +def docker_ids(override): + ids = {} + for service in ("api", "postgres", "lb"): + ids[service] = compose("ps", "-q", service, override=override).splitlines() + return ids + + +def stats_sample(ids): + all_ids = [id for service_ids in ids.values() for id in service_ids] + if not all_ids: + return {} + output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids) + by_id = {} + for line in output.splitlines(): + item = json.loads(line) + by_id[item["ID"]] = item + result = {} + for service, service_ids in ids.items(): + cpu = [] + memory = [] + for id in service_ids: + item = by_id.get(id[:12]) or by_id.get(id) + if not item: + raise ValueError("Docker stats omitted a benchmark container") + cpu.append(float(item["CPUPerc"].rstrip("%"))) + amount = item["MemUsage"].split("/")[0].strip() + value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha()) + memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024, + "kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3)) + result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)} + return result + + +def open_loop(base_url, path, token, rate, duration, workers=64): + """Schedule arrivals by clock; latency includes queue wait and HTTP time.""" + address = urlsplit(base_url) + if address.scheme != "http": + raise ValueError("this local lab3 benchmark expects HTTP") + jobs = Queue(maxsize=256) + latencies = [] + errors = defaultdict(int) + lock = threading.Lock() + starts = time.monotonic() + cpu_start = time.process_time() + count = math.ceil(rate * duration) + + def worker(): + connection = None + while True: + due = jobs.get() + if due is None: + jobs.task_done() + break + try: + if connection is None: + connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4) + connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"}) + response = connection.getresponse() + payload = response.read() + if response.status != 200: + with lock: + errors[f"http_{response.status}"] += 1 + elif len(json.loads(payload)["items"]) != 20: + with lock: + errors["wrong_page"] += 1 + else: + with lock: + latencies.append((time.monotonic() - due) * 1000) + except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException): + with lock: + errors["transport_or_response"] += 1 + if connection: + connection.close() + connection = None + finally: + jobs.task_done() + if connection: + connection.close() + + threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)] + for thread in threads: + thread.start() + dropped = 0 + for index in range(count): + due = starts + index / rate + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + try: + jobs.put_nowait(due) + except Full: + dropped += 1 + jobs.join() + for _ in threads: + jobs.put(None) + for thread in threads: + thread.join() + elapsed = time.monotonic() - starts + error_count = sum(errors.values()) + dropped + return { + "offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies), + "failed_requests": error_count, "error_ratio": round(error_count / count, 6), + "successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3), + "successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)}, + "errors_by_type": dict(errors), "queue_dropped": dropped, + "client_cpu_seconds": round(time.process_time() - cpu_start, 3), + } + + +def fixture(base_url, size, fixture_file): + if fixture_file.exists(): + saved = json.loads(fixture_file.read_text()) + if saved["base_url"] != base_url or saved["dataset_messages"] != size: + raise ValueError("saved fixture URL or size differs from requested profile") + Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]}) + return saved["token"], saved["path"] + client = Client(base_url) + alice, _, _, chat = client.bootstrap() + for index in range(size): + client.message(alice, chat, f"synthetic-scale-{index}") + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20" + fixture_file.parent.mkdir(parents=True, exist_ok=True) + with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file: + file.write(json.dumps({"base_url": base_url, "dataset_messages": size, + "token": alice["token"], "path": path}) + "\n") + return alice["token"], path + + +def run(args): + full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4] + if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20: + raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages") + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.output.exists(): + raise ValueError(f"output already exists: {args.output}") + token = path = None + rows = [] + with tempfile.TemporaryDirectory() as temporary: + override = Path(temporary) / "scale.yaml" + for mode in args.modes: + for replicas in args.replicas: + _, cpu, memory = topology(mode, replicas) + override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n") + compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override) + if token is None: + token, path = fixture(args.base_url, args.dataset_messages, args.fixture) + ids = docker_ids(override) + for rate in args.rates: + for repeat in range(1, args.repeats + 1): + open_loop(args.base_url, path, token, rate, args.warmup) + pool_before = pool_snapshot(args.base_url, replicas) + db_before = db_snapshot(override) + samples = [] + sampling_errors = [] + stop = threading.Event() + + def sample(): + while not stop.is_set(): + try: + samples.append(stats_sample(ids)) + except (subprocess.CalledProcessError, ValueError, KeyError) as error: + sampling_errors.append(type(error).__name__) + return + stop.wait(1) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + result = open_loop(args.base_url, path, token, rate, args.duration) + stop.set() + sampler.join() + if sampling_errors or not samples: + raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}") + db_after = db_snapshot(override) + if db_after["messages"] != db_before["messages"]: + raise RuntimeError("message dataset changed during read-only benchmark") + pool_after = pool_snapshot(args.base_url, replicas) + pool_delta = { + "observed_replicas": len(set(pool_before) & set(pool_after)), + "requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()), + "wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()), + "max_size": max((v["size"] for v in pool_after.values()), default=0), + } + services = {name: { + "avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3), + "max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3), + } for name in ("api", "postgres", "lb") if any(name in s for s in samples)} + row = { + "mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat, + "duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages, + "api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8, + **result, "docker": services, "pool": pool_delta, + "postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"}, + "database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"], + } + rows.append(row) + with args.output.open("a") as output: + output.write(json.dumps(row, ensure_ascii=False) + "\n") + print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True) + if full_matrix: + validate_matrix(rows) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500]) + parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"]) + parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4]) + parser.add_argument("--duration", type=int, default=8) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--dataset-messages", type=int, default=1000) + parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json") + parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl") + args = parser.parse_args() + if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or + any(mode not in ("fixed", "growth") for mode in args.modes) or + any(replica not in (1, 2, 4) for replica in args.replicas) or + args.duration <= 0 or args.warmup <= 0): + raise ValueError("rates and durations must be positive") + run(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100755 index 00000000..d9c755cb --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/benchmark.py "$@" diff --git a/scripts/plot_scale.py b/scripts/plot_scale.py new file mode 100644 index 00000000..87e58612 --- /dev/null +++ b/scripts/plot_scale.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Render the three-repeat scale matrix as small, dependency-free SVG plots.""" + +import argparse +import csv +import json +from pathlib import Path +import statistics + +from benchmark import validate_matrix + + +COLORS = {1: "#2563eb", 2: "#16a34a", 4: "#dc2626"} + + +def svg(mode, metric, rows, output): + width, height = 720, 420 + left, right, top, bottom = 75, 25, 35, 65 + plot_w, plot_h = width - left - right, height - top - bottom + rates = sorted({row["rate"] for row in rows}) + max_y = max((row[metric] or 0 for row in rows), default=0) * 1.1 or 1 + x = lambda rate: left + (rate - rates[0]) / (rates[-1] - rates[0]) * plot_w + y = lambda value: top + plot_h * (1 - value / max_y) + label = "Успешные запросы/с" if metric == "successful_rps" else "p95, мс" + parts = [f'', + '', + f'{mode}: {label}'] + for step in range(5): + value = max_y * step / 4 + py = y(value) + parts += [f'', + f'{value:.0f}'] + for rate in rates: + px = x(rate) + parts += [f'{rate}'] + for replicas in (1, 2, 4): + points = [(row["rate"], row[metric]) for row in rows if row["replicas"] == replicas and row[metric] is not None] + path = " ".join(f'{"M" if index == 0 else "L"}{x(rate):.1f},{y(value):.1f}' for index, (rate, value) in enumerate(points)) + parts.append(f'') + for rate, value in points: + parts.append(f'') + legend_x = left + 180 * (replicas == 2) + 360 * (replicas == 4) + parts.append(f'{replicas} API') + parts += [f'Заданная интенсивность, запросов/с', ''] + output.write_text("\n".join(parts) + "\n", encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + args = parser.parse_args() + rows = [json.loads(line) for line in args.input.read_text().splitlines() if line.strip()] + validate_matrix(rows) + summary = [] + for mode in ("fixed", "growth"): + for replicas in (1, 2, 4): + for rate in sorted({row["rate"] for row in rows}): + group = [row for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)] + def latency(p): + values = [row["successful_latency_ms"][p] for row in group if row["successful_latency_ms"][p] is not None] + return round(statistics.median(values), 3) if values else None + + summary.append({"mode": mode, "replicas": replicas, "rate": rate, + "successful_rps": round(statistics.median(row["successful_rps"] for row in group), 3), + "error_ratio": round(statistics.median(row["error_ratio"] for row in group), 6), + "p50_ms": latency("p50"), "p95_ms": latency("p95"), "p99_ms": latency("p99"), + "api_cpu_percent": round(statistics.median(row["docker"]["api"]["avg_cpu_percent"] for row in group), 3), + "postgres_cpu_percent": round(statistics.median(row["docker"]["postgres"]["avg_cpu_percent"] for row in group), 3), + "lb_cpu_percent": round(statistics.median(row["docker"]["lb"]["avg_cpu_percent"] for row in group), 3), + "api_memory_mib": round(statistics.median(row["docker"]["api"]["max_memory_mib"] for row in group), 3), + "postgres_memory_mib": round(statistics.median(row["docker"]["postgres"]["max_memory_mib"] for row in group), 3), + "lb_memory_mib": round(statistics.median(row["docker"]["lb"]["max_memory_mib"] for row in group), 3), + "pool_wait_ms": round(statistics.median(row["pool"]["wait_ms"] for row in group), 3), + "queue_dropped": statistics.median(row["queue_dropped"] for row in group)}) + with (args.input.parent / "summary.csv").open("w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=summary[0].keys(), lineterminator="\n") + writer.writeheader() + writer.writerows(summary) + for mode in ("fixed", "growth"): + group = [row for row in summary if row["mode"] == mode] + svg(mode, "successful_rps", group, args.input.parent / f"{mode}-rps.svg") + svg(mode, "p95_ms", group, args.input.parent / f"{mode}-p95.svg") + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py index e9b7a8f8..b4418118 100644 --- a/src/api.py +++ b/src/api.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from .auth import basic_credentials, bearer_token +from . import db from .models import CreateChat, CreateMessage, CreateUser from .store import store @@ -18,8 +19,14 @@ def live(): @router.get("/health/ready") -def ready(): - return store.ready() +def ready(response: Response): + result = store.ready() + stats = db.pool().get_stats() + for header, key in (("X-Pool-Requests", "requests_num"), ("X-Pool-Queued", "requests_queued"), + ("X-Pool-Wait-Ms", "requests_wait_ms"), ("X-Pool-Size", "pool_size"), + ("X-Pool-Available", "pool_available")): + response.headers[header] = str(stats.get(key, 0)) + return result @router.post("/api/v1/users", status_code=201) diff --git a/student-tests/test_benchmark_profile.py b/student-tests/test_benchmark_profile.py new file mode 100644 index 00000000..660dea8e --- /dev/null +++ b/student-tests/test_benchmark_profile.py @@ -0,0 +1,72 @@ +"""The scaling matrix must compare the same workload and resource budgets.""" + +import sys +import json +import unittest +from unittest.mock import patch +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from benchmark import open_loop, topology, validate_matrix + + +class BenchmarkProfileTests(unittest.TestCase): + def test_fixed_budget_is_identical_across_replica_counts(self): + budgets = [topology("fixed", replicas) for replicas in (1, 2, 4)] + self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [1.0] * 3) + self.assertEqual([replicas * memory for replicas, _, memory in budgets], [512] * 3) + + def test_growth_budget_increases_with_replicas(self): + budgets = [topology("growth", replicas) for replicas in (1, 2, 4)] + self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [0.5, 1.0, 2.0]) + self.assertEqual([replicas * memory for replicas, _, memory in budgets], [256, 512, 1024]) + + def test_matrix_requires_three_repeats_and_same_workload(self): + rows = [ + {"mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat, + "duration": 8, "warmup": 2, "dataset_messages": 1000, + "api_cpu_each": topology(mode, replicas)[1], + "api_memory_mib_each": topology(mode, replicas)[2]} + for mode in ("fixed", "growth") + for replicas in (1, 2, 4) + for rate in (100, 250, 500) + for repeat in (1, 2, 3) + ] + validate_matrix(rows) + with self.assertRaises(ValueError): + validate_matrix(rows[:-1]) + rows[-1]["dataset_messages"] = 999 + with self.assertRaises(ValueError): + validate_matrix(rows) + + def test_open_loop_counts_responses_at_a_scheduled_rate(self): + class Response: + status = 200 + + def read(self): + return json.dumps({"items": [{}] * 20}).encode() + + class Connection: + def __init__(self, *args, **kwargs): + pass + + def request(self, *args, **kwargs): + pass + + def getresponse(self): + return Response() + + def close(self): + pass + + with patch("benchmark.http.client.HTTPConnection", Connection): + result = open_loop("http://127.0.0.1:8080", "/history", "synthetic", 20, 0.2, workers=2) + self.assertEqual(result["scheduled_requests"], 4) + self.assertEqual(result["successful_requests"], 4) + self.assertEqual(result["failed_requests"], 0) + self.assertGreater(result["successful_latency_ms"]["p95"], 0) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 9e18e2f07c3c7ae4add426d1ada33cdaed1942b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:02:24 +0300 Subject: [PATCH 8/9] docs: Remove trailing spaces from failover evidence --- evidence/failover-u4-first.txt | 6 +++--- evidence/topology-u4.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evidence/failover-u4-first.txt b/evidence/failover-u4-first.txt index 866b821a..03b2f602 100644 --- a/evidence/failover-u4-first.txt +++ b/evidence/failover-u4-first.txt @@ -1,6 +1,6 @@ Commands: tests/load.py --duration 20 --concurrency 4, make test-failover ACTION_SCRIPT=scripts/stop-one.sh Failover exit: 0 -python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8083" +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8083" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. {"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 3.623} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. @@ -25,7 +25,7 @@ Load exit: 0 Actual containers after hook: NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 10 minutes ago Up About a minute (healthy) 8080/tcp -mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 9 minutes ago Exited (0) 16 seconds ago +mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 9 minutes ago Exited (0) 16 seconds ago mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 23 minutes 127.0.0.1:8083->8080/tcp -mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 18 minutes ago Exited (0) 57 seconds ago +mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 18 minutes ago Exited (0) 57 seconds ago mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 18 minutes ago Up 18 minutes (healthy) 5432/tcp diff --git a/evidence/topology-u4.txt b/evidence/topology-u4.txt index e403a36c..a3eab65d 100644 --- a/evidence/topology-u4.txt +++ b/evidence/topology-u4.txt @@ -13,5 +13,5 @@ NAME IMAGE COMMAND SER mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 15 minutes 127.0.0.1:8083->8080/tcp -mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 10 minutes ago Exited (0) 3 minutes ago +mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 10 minutes ago Exited (0) 3 minutes ago mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 10 minutes ago Up 10 minutes (healthy) 5432/tcp -- 2.54.0 From a46a04b86659c263de1ad1d90ce9bd8ac189d797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 7 Nov 2026 09:53:49 +0300 Subject: [PATCH 9/9] fix: preserve NUL characters in persisted messages Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. (cherry picked from commit 386276532730b8758298080b49db63738ca300a4) --- migrations/009_message_text_utf8.sql | 6 + src/store.py | 4 +- student-tests/test_database.py | 2 +- student-tests/test_idempotency.py | 2 +- student-tests/test_message_unicode.py | 156 ++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 migrations/009_message_text_utf8.sql create mode 100644 student-tests/test_message_unicode.py diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 00000000..86196ee6 --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/src/store.py b/src/store.py index d91a5644..0db641b1 100644 --- a/src/store.py +++ b/src/store.py @@ -78,7 +78,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), } @@ -240,7 +240,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() conn.execute( "UPDATE message_replays SET message_id=%s WHERE user_id=%s AND chat_id=%s AND request_key=%s", diff --git a/student-tests/test_database.py b/student-tests/test_database.py index 132318f9..61a282f2 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -30,7 +30,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,), (3,)]) + self.assertEqual(versions, [(1,), (2,), (3,), (9,)]) name = "u_" + uuid4().hex[:16] def register(_): diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py index dc52711b..ebc1c99f 100644 --- a/student-tests/test_idempotency.py +++ b/student-tests/test_idempotency.py @@ -38,7 +38,7 @@ class IdempotencyTests(unittest.TestCase): with psycopg.connect(URL) as conn: count = conn.execute( "SELECT count(*) FROM messages WHERE chat_id=%s AND sender_id=%s AND text=%s", - (chat["id"], alice["id"], "same body"), + (chat["id"], alice["id"], b"same body"), ).fetchone()[0] self.assertEqual(count, 1) diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 00000000..4ff79d81 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() -- 2.54.0