Bound PostgreSQL response waits and test paused database

This commit is contained in:
Polusikkkk
2026-11-06 18:44:46 +03:00
parent 0f6615dbb9
commit dcaebee5f1
4 changed files with 288 additions and 1 deletions
+38
View File
@@ -457,3 +457,41 @@ p50/p95/p99 и CPU не использовались как критерий у
- Linux Kernel Documentation: /proc, cgroup v2 memory counters.
- Python-библиотеки: FastAPI, Uvicorn, psycopg, psycopg-pool,
Redis client, argon2-cffi, boto3, Pillow.
## Исправление по замечанию: приостановка PostgreSQL
Проблема: при приостановке PostgreSQL запрос истории мог ждать
до клиентского таймаута. Серверный statement_timeout не обеспечивал
ограничение ожидания при замороженных процессах БД.
В app/db.py добавлен DeadlineConnection, используемый пулом соединений.
Ожидание отдельной операции соединения ограничено на стороне приложения
значением DB_IO_TIMEOUT_SECONDS, по умолчанию 3 секунды.
При превышении срока соединение закрывается и поднимается OperationalError.
Существующий обработчик app/main.py возвращает 503 unavailable.
Реализация использует низкоуровневое ожидание Psycopg;
проверена с закреплёнными psycopg[binary]==3.2.6
и psycopg-pool==3.2.6. При обновлении драйвера тест необходимо повторить.
Команда проверки:
```sh
python3 student-tests/postgres_pause.py
```
Фактический результат 06.11.2026:
- До отказа история доступна через обе API-реплики.
- PostgreSQL приостановлен через docker pause.
- Четыре параллельных запроса истории получили 503 unavailable.
- Ответы пришли от обеих API-реплик с X-Instance-Id.
- Время ответа: 3,0871–3,0881 секунды.
- PostgreSQL возобновлён в finally.
- История восстановилась, подтверждённое сообщение сохранилось.
- Итог: PASS: bounded API 503 on both replicas; history recovered.
Машинное доказательство: evidence/postgres_pause.json.
Тест проверяет HTTP-ответ при приостановленной БД с порогом 6 секунд.
Трёхсекундное ограничение отдельного ожидания не является универсальной
гарантией трёхсекундного времени любого составного HTTP-запроса.
+33 -1
View File
@@ -1,13 +1,45 @@
from __future__ import annotations
import asyncio
import os
from psycopg import AsyncConnection, OperationalError, waiting
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
DB_IO_TIMEOUT_SECONDS = float(
os.getenv("DB_IO_TIMEOUT_SECONDS", "3")
)
if DB_IO_TIMEOUT_SECONDS <= 0:
raise ValueError("DB_IO_TIMEOUT_SECONDS must be positive")
class DeadlineConnection(AsyncConnection):
async def wait(self, gen, interval=0.1):
try:
async with asyncio.timeout(DB_IO_TIMEOUT_SECONDS):
return await waiting.wait_async(
gen,
self.pgconn.socket,
interval=interval,
)
except TimeoutError:
await self.close()
raise OperationalError(
"PostgreSQL response deadline exceeded"
) from None
except asyncio.CancelledError:
await self.close()
raise
pool = AsyncConnectionPool(
conninfo="",
connection_class=DeadlineConnection,
kwargs={
"host": os.getenv("PGHOST", "postgres"),
"port": int(os.getenv("PGPORT", "5432")),
@@ -41,4 +73,4 @@ async def close_db() -> None:
async def check_db() -> None:
async with pool.connection() as conn:
await conn.execute("SELECT 1")
await conn.execute("SELECT 1")
+41
View File
@@ -0,0 +1,41 @@
{
"scenario": "postgres_pause_history",
"passed": true,
"max_response_seconds": 6.0,
"baseline_instances": [
"3dbd75387914:92d7529650bf",
"b9966df7dc8d:a4e095d4b95b"
],
"paused_requests": [
{
"ok": true,
"status": 503,
"code": "unavailable",
"instance": "b9966df7dc8d:a4e095d4b95b",
"elapsed_s": 3.0877
},
{
"ok": true,
"status": 503,
"code": "unavailable",
"instance": "3dbd75387914:92d7529650bf",
"elapsed_s": 3.0881
},
{
"ok": true,
"status": 503,
"code": "unavailable",
"instance": "3dbd75387914:92d7529650bf",
"elapsed_s": 3.088
},
{
"ok": true,
"status": 503,
"code": "unavailable",
"instance": "b9966df7dc8d:a4e095d4b95b",
"elapsed_s": 3.0871
}
],
"recovery_observed_after_s": 1.0124,
"confirmed_message_preserved": true
}
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
from __future__ import annotations
import concurrent.futures
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
BASE_URL = "http://127.0.0.1:18082"
MAX_RESPONSE_SECONDS = 6.0
WATCHDOG_SECONDS = 8.0
COMPOSE = [
"docker", "compose",
"-f", "compose.yaml",
"-f", "compose.s3.yaml",
"--env-file", ".env",
]
def docker(*args):
return subprocess.check_output(
["docker", *args],
cwd=ROOT,
text=True,
timeout=20,
stderr=subprocess.PIPE,
).strip()
def main():
pg_id = subprocess.check_output(
[*COMPOSE, "ps", "-q", "postgres"],
cwd=ROOT,
text=True,
timeout=20,
).strip()
assert pg_id, "PostgreSQL container not found"
assert docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
) == "false", "PostgreSQL is already paused"
c = Client(BASE_URL, None)
alice, _, _, chat = c.bootstrap()
message = c.message(alice, chat, "postgres-pause regression")["json"]
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
baseline_instances = set()
for _ in range(6):
response = c.call(
"GET", path, actor=alice, schema="MessagePage"
)
assert any(
item["id"] == message["id"]
for item in response["json"]["items"]
), "Baseline message is missing"
instance = response["headers"].get("X-Instance-Id")
assert instance, "Baseline response has no X-Instance-Id"
baseline_instances.add(instance)
assert len(baseline_instances) >= 2, (
"Baseline did not reach both API replicas"
)
print("Baseline: history is available through both replicas", flush=True)
def probe():
started = time.monotonic()
try:
client = Client(BASE_URL, None)
response = client.call("GET", path, 503, actor=alice)
elapsed = time.monotonic() - started
instance = response["headers"].get("X-Instance-Id")
code = response["json"]["error"]["code"]
return {
"ok": bool(instance) and code == "unavailable",
"status": 503,
"code": code,
"instance": instance,
"elapsed_s": round(elapsed, 4),
}
except Exception as exc:
return {
"ok": False,
"error_type": type(exc).__name__,
"elapsed_s": round(time.monotonic() - started, 4),
}
executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
paused_by_test = False
try:
paused_by_test = True
docker("pause", pg_id)
assert docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
) == "true"
print("PostgreSQL paused; requesting history", flush=True)
futures = [executor.submit(probe) for _ in range(4)]
_, pending = concurrent.futures.wait(
futures, timeout=WATCHDOG_SECONDS
)
assert not pending, "History requests exceeded the test watchdog"
results = [future.result() for future in futures]
print(json.dumps(results, indent=2), flush=True)
assert all(item["ok"] for item in results), (
"Expected API JSON 503 unavailable with X-Instance-Id"
)
assert all(
item["elapsed_s"] < MAX_RESPONSE_SECONDS
for item in results
), "API response exceeded the declared deadline"
failed_instances = {item["instance"] for item in results}
assert baseline_instances.issubset(failed_instances), (
"Paused-DB requests did not exercise both baseline replicas"
)
finally:
try:
if paused_by_test:
state = docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
)
if state == "true":
docker("unpause", pg_id)
print("PostgreSQL unpaused", flush=True)
finally:
executor.shutdown(wait=False, cancel_futures=True)
recovery_started = time.monotonic()
recovery_deadline = recovery_started + 60
while True:
try:
response = c.call(
"GET", path, actor=alice, schema="MessagePage"
)
assert any(
item["id"] == message["id"]
for item in response["json"]["items"]
), "Confirmed message disappeared"
break
except Exception:
if time.monotonic() >= recovery_deadline:
raise AssertionError("History did not recover") from None
time.sleep(1)
report = {
"scenario": "postgres_pause_history",
"passed": True,
"max_response_seconds": MAX_RESPONSE_SECONDS,
"baseline_instances": sorted(baseline_instances),
"paused_requests": results,
"recovery_observed_after_s": round(
time.monotonic() - recovery_started, 4
),
"confirmed_message_preserved": True,
}
target = ROOT / "evidence" / "postgres_pause.json"
target.parent.mkdir(exist_ok=True)
target.write_text(json.dumps(report, indent=2) + "\n")
print("PASS: bounded API 503 on both replicas; history recovered", flush=True)
if __name__ == "__main__":
main()