Bound PostgreSQL response waits and test paused database
This commit is contained in:
@@ -457,3 +457,41 @@ p50/p95/p99 и CPU не использовались как критерий у
|
||||
- Linux Kernel Documentation: /proc, cgroup v2 memory counters.
|
||||
- Python-библиотеки: FastAPI, Uvicorn, psycopg, psycopg-pool,
|
||||
Redis client, argon2-cffi, boto3, Pillow.
|
||||
|
||||
## Исправление по замечанию: приостановка PostgreSQL
|
||||
|
||||
Проблема: при приостановке PostgreSQL запрос истории мог ждать
|
||||
до клиентского таймаута. Серверный statement_timeout не обеспечивал
|
||||
ограничение ожидания при замороженных процессах БД.
|
||||
|
||||
В app/db.py добавлен DeadlineConnection, используемый пулом соединений.
|
||||
Ожидание отдельной операции соединения ограничено на стороне приложения
|
||||
значением DB_IO_TIMEOUT_SECONDS, по умолчанию 3 секунды.
|
||||
При превышении срока соединение закрывается и поднимается OperationalError.
|
||||
Существующий обработчик app/main.py возвращает 503 unavailable.
|
||||
|
||||
Реализация использует низкоуровневое ожидание Psycopg;
|
||||
проверена с закреплёнными psycopg[binary]==3.2.6
|
||||
и psycopg-pool==3.2.6. При обновлении драйвера тест необходимо повторить.
|
||||
|
||||
Команда проверки:
|
||||
|
||||
```sh
|
||||
python3 student-tests/postgres_pause.py
|
||||
```
|
||||
|
||||
Фактический результат 06.11.2026:
|
||||
|
||||
- До отказа история доступна через обе API-реплики.
|
||||
- PostgreSQL приостановлен через docker pause.
|
||||
- Четыре параллельных запроса истории получили 503 unavailable.
|
||||
- Ответы пришли от обеих API-реплик с X-Instance-Id.
|
||||
- Время ответа: 3,0871–3,0881 секунды.
|
||||
- PostgreSQL возобновлён в finally.
|
||||
- История восстановилась, подтверждённое сообщение сохранилось.
|
||||
- Итог: PASS: bounded API 503 on both replicas; history recovered.
|
||||
|
||||
Машинное доказательство: evidence/postgres_pause.json.
|
||||
Тест проверяет HTTP-ответ при приостановленной БД с порогом 6 секунд.
|
||||
Трёхсекундное ограничение отдельного ожидания не является универсальной
|
||||
гарантией трёхсекундного времени любого составного HTTP-запроса.
|
||||
|
||||
@@ -1,13 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from psycopg import AsyncConnection, OperationalError, waiting
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
|
||||
DB_IO_TIMEOUT_SECONDS = float(
|
||||
os.getenv("DB_IO_TIMEOUT_SECONDS", "3")
|
||||
)
|
||||
|
||||
if DB_IO_TIMEOUT_SECONDS <= 0:
|
||||
raise ValueError("DB_IO_TIMEOUT_SECONDS must be positive")
|
||||
|
||||
|
||||
class DeadlineConnection(AsyncConnection):
|
||||
async def wait(self, gen, interval=0.1):
|
||||
try:
|
||||
async with asyncio.timeout(DB_IO_TIMEOUT_SECONDS):
|
||||
return await waiting.wait_async(
|
||||
gen,
|
||||
self.pgconn.socket,
|
||||
interval=interval,
|
||||
)
|
||||
|
||||
except TimeoutError:
|
||||
await self.close()
|
||||
raise OperationalError(
|
||||
"PostgreSQL response deadline exceeded"
|
||||
) from None
|
||||
|
||||
except asyncio.CancelledError:
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
|
||||
pool = AsyncConnectionPool(
|
||||
conninfo="",
|
||||
connection_class=DeadlineConnection,
|
||||
kwargs={
|
||||
"host": os.getenv("PGHOST", "postgres"),
|
||||
"port": int(os.getenv("PGPORT", "5432")),
|
||||
@@ -41,4 +73,4 @@ async def close_db() -> None:
|
||||
|
||||
async def check_db() -> None:
|
||||
async with pool.connection() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
await conn.execute("SELECT 1")
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"scenario": "postgres_pause_history",
|
||||
"passed": true,
|
||||
"max_response_seconds": 6.0,
|
||||
"baseline_instances": [
|
||||
"3dbd75387914:92d7529650bf",
|
||||
"b9966df7dc8d:a4e095d4b95b"
|
||||
],
|
||||
"paused_requests": [
|
||||
{
|
||||
"ok": true,
|
||||
"status": 503,
|
||||
"code": "unavailable",
|
||||
"instance": "b9966df7dc8d:a4e095d4b95b",
|
||||
"elapsed_s": 3.0877
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"status": 503,
|
||||
"code": "unavailable",
|
||||
"instance": "3dbd75387914:92d7529650bf",
|
||||
"elapsed_s": 3.0881
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"status": 503,
|
||||
"code": "unavailable",
|
||||
"instance": "3dbd75387914:92d7529650bf",
|
||||
"elapsed_s": 3.088
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"status": 503,
|
||||
"code": "unavailable",
|
||||
"instance": "b9966df7dc8d:a4e095d4b95b",
|
||||
"elapsed_s": 3.0871
|
||||
}
|
||||
],
|
||||
"recovery_observed_after_s": 1.0124,
|
||||
"confirmed_message_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client
|
||||
|
||||
|
||||
BASE_URL = "http://127.0.0.1:18082"
|
||||
MAX_RESPONSE_SECONDS = 6.0
|
||||
WATCHDOG_SECONDS = 8.0
|
||||
|
||||
COMPOSE = [
|
||||
"docker", "compose",
|
||||
"-f", "compose.yaml",
|
||||
"-f", "compose.s3.yaml",
|
||||
"--env-file", ".env",
|
||||
]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(
|
||||
["docker", *args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
timeout=20,
|
||||
stderr=subprocess.PIPE,
|
||||
).strip()
|
||||
|
||||
|
||||
def main():
|
||||
pg_id = subprocess.check_output(
|
||||
[*COMPOSE, "ps", "-q", "postgres"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
timeout=20,
|
||||
).strip()
|
||||
assert pg_id, "PostgreSQL container not found"
|
||||
assert docker(
|
||||
"inspect", "-f", "{{.State.Paused}}", pg_id
|
||||
) == "false", "PostgreSQL is already paused"
|
||||
|
||||
c = Client(BASE_URL, None)
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
message = c.message(alice, chat, "postgres-pause regression")["json"]
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
|
||||
baseline_instances = set()
|
||||
for _ in range(6):
|
||||
response = c.call(
|
||||
"GET", path, actor=alice, schema="MessagePage"
|
||||
)
|
||||
assert any(
|
||||
item["id"] == message["id"]
|
||||
for item in response["json"]["items"]
|
||||
), "Baseline message is missing"
|
||||
instance = response["headers"].get("X-Instance-Id")
|
||||
assert instance, "Baseline response has no X-Instance-Id"
|
||||
baseline_instances.add(instance)
|
||||
|
||||
assert len(baseline_instances) >= 2, (
|
||||
"Baseline did not reach both API replicas"
|
||||
)
|
||||
print("Baseline: history is available through both replicas", flush=True)
|
||||
|
||||
def probe():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
client = Client(BASE_URL, None)
|
||||
response = client.call("GET", path, 503, actor=alice)
|
||||
elapsed = time.monotonic() - started
|
||||
instance = response["headers"].get("X-Instance-Id")
|
||||
code = response["json"]["error"]["code"]
|
||||
return {
|
||||
"ok": bool(instance) and code == "unavailable",
|
||||
"status": 503,
|
||||
"code": code,
|
||||
"instance": instance,
|
||||
"elapsed_s": round(elapsed, 4),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_type": type(exc).__name__,
|
||||
"elapsed_s": round(time.monotonic() - started, 4),
|
||||
}
|
||||
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
|
||||
paused_by_test = False
|
||||
try:
|
||||
paused_by_test = True
|
||||
docker("pause", pg_id)
|
||||
assert docker(
|
||||
"inspect", "-f", "{{.State.Paused}}", pg_id
|
||||
) == "true"
|
||||
print("PostgreSQL paused; requesting history", flush=True)
|
||||
|
||||
futures = [executor.submit(probe) for _ in range(4)]
|
||||
_, pending = concurrent.futures.wait(
|
||||
futures, timeout=WATCHDOG_SECONDS
|
||||
)
|
||||
assert not pending, "History requests exceeded the test watchdog"
|
||||
|
||||
results = [future.result() for future in futures]
|
||||
print(json.dumps(results, indent=2), flush=True)
|
||||
|
||||
assert all(item["ok"] for item in results), (
|
||||
"Expected API JSON 503 unavailable with X-Instance-Id"
|
||||
)
|
||||
assert all(
|
||||
item["elapsed_s"] < MAX_RESPONSE_SECONDS
|
||||
for item in results
|
||||
), "API response exceeded the declared deadline"
|
||||
|
||||
failed_instances = {item["instance"] for item in results}
|
||||
assert baseline_instances.issubset(failed_instances), (
|
||||
"Paused-DB requests did not exercise both baseline replicas"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
if paused_by_test:
|
||||
state = docker(
|
||||
"inspect", "-f", "{{.State.Paused}}", pg_id
|
||||
)
|
||||
if state == "true":
|
||||
docker("unpause", pg_id)
|
||||
print("PostgreSQL unpaused", flush=True)
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
recovery_started = time.monotonic()
|
||||
recovery_deadline = recovery_started + 60
|
||||
while True:
|
||||
try:
|
||||
response = c.call(
|
||||
"GET", path, actor=alice, schema="MessagePage"
|
||||
)
|
||||
assert any(
|
||||
item["id"] == message["id"]
|
||||
for item in response["json"]["items"]
|
||||
), "Confirmed message disappeared"
|
||||
break
|
||||
except Exception:
|
||||
if time.monotonic() >= recovery_deadline:
|
||||
raise AssertionError("History did not recover") from None
|
||||
time.sleep(1)
|
||||
|
||||
report = {
|
||||
"scenario": "postgres_pause_history",
|
||||
"passed": True,
|
||||
"max_response_seconds": MAX_RESPONSE_SECONDS,
|
||||
"baseline_instances": sorted(baseline_instances),
|
||||
"paused_requests": results,
|
||||
"recovery_observed_after_s": round(
|
||||
time.monotonic() - recovery_started, 4
|
||||
),
|
||||
"confirmed_message_preserved": True,
|
||||
}
|
||||
target = ROOT / "evidence" / "postgres_pause.json"
|
||||
target.parent.mkdir(exist_ok=True)
|
||||
target.write_text(json.dumps(report, indent=2) + "\n")
|
||||
|
||||
print("PASS: bounded API 503 on both replicas; history recovered", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user