diff --git a/REPORT.md b/REPORT.md index b4900aa7..4748e318 100644 --- a/REPORT.md +++ b/REPORT.md @@ -457,3 +457,41 @@ p50/p95/p99 и CPU не использовались как критерий у - Linux Kernel Documentation: /proc, cgroup v2 memory counters. - Python-библиотеки: FastAPI, Uvicorn, psycopg, psycopg-pool, Redis client, argon2-cffi, boto3, Pillow. + +## Исправление по замечанию: приостановка PostgreSQL + +Проблема: при приостановке PostgreSQL запрос истории мог ждать +до клиентского таймаута. Серверный statement_timeout не обеспечивал +ограничение ожидания при замороженных процессах БД. + +В app/db.py добавлен DeadlineConnection, используемый пулом соединений. +Ожидание отдельной операции соединения ограничено на стороне приложения +значением DB_IO_TIMEOUT_SECONDS, по умолчанию 3 секунды. +При превышении срока соединение закрывается и поднимается OperationalError. +Существующий обработчик app/main.py возвращает 503 unavailable. + +Реализация использует низкоуровневое ожидание Psycopg; +проверена с закреплёнными psycopg[binary]==3.2.6 +и psycopg-pool==3.2.6. При обновлении драйвера тест необходимо повторить. + +Команда проверки: + +```sh +python3 student-tests/postgres_pause.py +``` + +Фактический результат 06.11.2026: + +- До отказа история доступна через обе API-реплики. +- PostgreSQL приостановлен через docker pause. +- Четыре параллельных запроса истории получили 503 unavailable. +- Ответы пришли от обеих API-реплик с X-Instance-Id. +- Время ответа: 3,0871–3,0881 секунды. +- PostgreSQL возобновлён в finally. +- История восстановилась, подтверждённое сообщение сохранилось. +- Итог: PASS: bounded API 503 on both replicas; history recovered. + +Машинное доказательство: evidence/postgres_pause.json. +Тест проверяет HTTP-ответ при приостановленной БД с порогом 6 секунд. +Трёхсекундное ограничение отдельного ожидания не является универсальной +гарантией трёхсекундного времени любого составного HTTP-запроса. diff --git a/app/db.py b/app/db.py index 0cdd85ad..8aedf8d2 100644 --- a/app/db.py +++ b/app/db.py @@ -1,13 +1,45 @@ from __future__ import annotations +import asyncio import os +from psycopg import AsyncConnection, OperationalError, waiting from psycopg.rows import dict_row from psycopg_pool import AsyncConnectionPool +DB_IO_TIMEOUT_SECONDS = float( + os.getenv("DB_IO_TIMEOUT_SECONDS", "3") +) + +if DB_IO_TIMEOUT_SECONDS <= 0: + raise ValueError("DB_IO_TIMEOUT_SECONDS must be positive") + + +class DeadlineConnection(AsyncConnection): + async def wait(self, gen, interval=0.1): + try: + async with asyncio.timeout(DB_IO_TIMEOUT_SECONDS): + return await waiting.wait_async( + gen, + self.pgconn.socket, + interval=interval, + ) + + except TimeoutError: + await self.close() + raise OperationalError( + "PostgreSQL response deadline exceeded" + ) from None + + except asyncio.CancelledError: + await self.close() + raise + + pool = AsyncConnectionPool( conninfo="", + connection_class=DeadlineConnection, kwargs={ "host": os.getenv("PGHOST", "postgres"), "port": int(os.getenv("PGPORT", "5432")), @@ -41,4 +73,4 @@ async def close_db() -> None: async def check_db() -> None: async with pool.connection() as conn: - await conn.execute("SELECT 1") \ No newline at end of file + await conn.execute("SELECT 1") diff --git a/evidence/postgres_pause.json b/evidence/postgres_pause.json new file mode 100644 index 00000000..051b9aa8 --- /dev/null +++ b/evidence/postgres_pause.json @@ -0,0 +1,41 @@ +{ + "scenario": "postgres_pause_history", + "passed": true, + "max_response_seconds": 6.0, + "baseline_instances": [ + "3dbd75387914:92d7529650bf", + "b9966df7dc8d:a4e095d4b95b" + ], + "paused_requests": [ + { + "ok": true, + "status": 503, + "code": "unavailable", + "instance": "b9966df7dc8d:a4e095d4b95b", + "elapsed_s": 3.0877 + }, + { + "ok": true, + "status": 503, + "code": "unavailable", + "instance": "3dbd75387914:92d7529650bf", + "elapsed_s": 3.0881 + }, + { + "ok": true, + "status": 503, + "code": "unavailable", + "instance": "3dbd75387914:92d7529650bf", + "elapsed_s": 3.088 + }, + { + "ok": true, + "status": 503, + "code": "unavailable", + "instance": "b9966df7dc8d:a4e095d4b95b", + "elapsed_s": 3.0871 + } + ], + "recovery_observed_after_s": 1.0124, + "confirmed_message_preserved": true +} diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 00000000..018dad3e --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import concurrent.futures +import json +from pathlib import Path +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client + + +BASE_URL = "http://127.0.0.1:18082" +MAX_RESPONSE_SECONDS = 6.0 +WATCHDOG_SECONDS = 8.0 + +COMPOSE = [ + "docker", "compose", + "-f", "compose.yaml", + "-f", "compose.s3.yaml", + "--env-file", ".env", +] + + +def docker(*args): + return subprocess.check_output( + ["docker", *args], + cwd=ROOT, + text=True, + timeout=20, + stderr=subprocess.PIPE, + ).strip() + + +def main(): + pg_id = subprocess.check_output( + [*COMPOSE, "ps", "-q", "postgres"], + cwd=ROOT, + text=True, + timeout=20, + ).strip() + assert pg_id, "PostgreSQL container not found" + assert docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) == "false", "PostgreSQL is already paused" + + c = Client(BASE_URL, None) + alice, _, _, chat = c.bootstrap() + message = c.message(alice, chat, "postgres-pause regression")["json"] + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + + baseline_instances = set() + for _ in range(6): + response = c.call( + "GET", path, actor=alice, schema="MessagePage" + ) + assert any( + item["id"] == message["id"] + for item in response["json"]["items"] + ), "Baseline message is missing" + instance = response["headers"].get("X-Instance-Id") + assert instance, "Baseline response has no X-Instance-Id" + baseline_instances.add(instance) + + assert len(baseline_instances) >= 2, ( + "Baseline did not reach both API replicas" + ) + print("Baseline: history is available through both replicas", flush=True) + + def probe(): + started = time.monotonic() + try: + client = Client(BASE_URL, None) + response = client.call("GET", path, 503, actor=alice) + elapsed = time.monotonic() - started + instance = response["headers"].get("X-Instance-Id") + code = response["json"]["error"]["code"] + return { + "ok": bool(instance) and code == "unavailable", + "status": 503, + "code": code, + "instance": instance, + "elapsed_s": round(elapsed, 4), + } + except Exception as exc: + return { + "ok": False, + "error_type": type(exc).__name__, + "elapsed_s": round(time.monotonic() - started, 4), + } + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) + paused_by_test = False + try: + paused_by_test = True + docker("pause", pg_id) + assert docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) == "true" + print("PostgreSQL paused; requesting history", flush=True) + + futures = [executor.submit(probe) for _ in range(4)] + _, pending = concurrent.futures.wait( + futures, timeout=WATCHDOG_SECONDS + ) + assert not pending, "History requests exceeded the test watchdog" + + results = [future.result() for future in futures] + print(json.dumps(results, indent=2), flush=True) + + assert all(item["ok"] for item in results), ( + "Expected API JSON 503 unavailable with X-Instance-Id" + ) + assert all( + item["elapsed_s"] < MAX_RESPONSE_SECONDS + for item in results + ), "API response exceeded the declared deadline" + + failed_instances = {item["instance"] for item in results} + assert baseline_instances.issubset(failed_instances), ( + "Paused-DB requests did not exercise both baseline replicas" + ) + finally: + try: + if paused_by_test: + state = docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) + if state == "true": + docker("unpause", pg_id) + print("PostgreSQL unpaused", flush=True) + finally: + executor.shutdown(wait=False, cancel_futures=True) + + recovery_started = time.monotonic() + recovery_deadline = recovery_started + 60 + while True: + try: + response = c.call( + "GET", path, actor=alice, schema="MessagePage" + ) + assert any( + item["id"] == message["id"] + for item in response["json"]["items"] + ), "Confirmed message disappeared" + break + except Exception: + if time.monotonic() >= recovery_deadline: + raise AssertionError("History did not recover") from None + time.sleep(1) + + report = { + "scenario": "postgres_pause_history", + "passed": True, + "max_response_seconds": MAX_RESPONSE_SECONDS, + "baseline_instances": sorted(baseline_instances), + "paused_requests": results, + "recovery_observed_after_s": round( + time.monotonic() - recovery_started, 4 + ), + "confirmed_message_preserved": True, + } + target = ROOT / "evidence" / "postgres_pause.json" + target.parent.mkdir(exist_ok=True) + target.write_text(json.dumps(report, indent=2) + "\n") + + print("PASS: bounded API 503 on both replicas; history recovered", flush=True) + + +if __name__ == "__main__": + main()