diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ddf661a1..c6743d9f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,7 +2,7 @@ Автор / группа: Синюк Максим Павлович БПИ2301 Целевая оценка (3/4/5): 5 -Трек 5, если заявлен: TODO +Трек 1, если заявлен: "Цена масштабирования" Что реализовано и что перенесено: перенесены пользовательские тесты, валидация(перенастроена на пароль), роуты(перенастроены на защиту по токену сессии), модели ORM и работа с БД PostgreSQL, Redis кэширование Команды запуска из чистого clone: cd mtusi2026lab3 | docker compose -d --build | docker compose logs api --tail 50 @@ -10,8 +10,7 @@ Ограничения / известные ошибки: описано в REPORT.md Источники и использование ИИ: руками -Graceful Degradation: SIGTERM перестаёт принимать новые запросы, выполняет уже полученные в течение 30 секунд и останавливает процесс, встроенная функция сервера Uvicorn. Для проверки отказоустойчивости был использован скрипт `scripts/failover_test.sh`, который выполняет нагрузочное тестирование (`tests/load.py`) с помощью файла `scripts/test_idempotency_failover.py` и последовательно останавливает/восстанавливает(проверка на отказоустойчивость) реплики API. Результат описан в README.md и метрики представлены в `results/failover_metrics.json`. Идемпотентность при параллельных запросах с одинаковым idempotency-key проводилась с concurrency 4, duration 60s и при 2 репликах api с ограничениями cpus 0.5 и memory 512M. Cache Stampede при холодном запуске решается через Redis Lock на idempotency-key для конкретного запроса благодаря асинхронности запросов. Параллельные запросы с одним idempotent-key создают одну запись, response кэшируется и отдаётся без обращения к БД. Health/readiness и timeouts настроены в `docker-compose.yml` и `nginx.conf`, время обнаружения отказа инстанса ~20 сек. Миграции вынесены в отдельный сервис `migrate` в `docker-compose.yml`. Сервис `api` имеет зависимость `depends_on: migrate (condition: service_completed_successfully)`. В виду этого миграции выполняются только один раз перед запуском реплик api. -- Реплики API только проверяют подключение к БД и инициализируют Redis +Graceful Degradation: SIGTERM перестаёт принимать новые запросы, выполняет уже полученные в течение 30 секунд и останавливает процесс, встроенная функция сервера Uvicorn. Для проверки отказоустойчивости был использован скрипт `scripts/failover_test.sh`, который выполняет нагрузочное тестирование (`tests/load.py`) с помощью файла `scripts/test_idempotency_failover.py` и последовательно останавливает/восстанавливает(проверка на отказоустойчивость) реплики API. Результат описан в README.md и метрики представлены в `results/failover_metrics.json`. Идемпотентность при параллельных запросах с одинаковым idempotency-key проводилась с concurrency 4, duration 60s и при 2 репликах api с ограничениями cpus 0.5 и memory 512M. Cache Stampede при холодном запуске решается через Redis Lock на idempotency-key для конкретного запроса благодаря асинхронности запросов. Параллельные запросы с одним idempotent-key создают одну запись, response кэшируется и отдаётся без обращения к БД. Health/readiness и timeouts настроены в `docker-compose.yml` и `nginx.conf`, время обнаружения отказа инстанса ~20 сек. Миграции вынесены в отдельный сервис `migrate` в `docker-compose.yml`. Сервис `api` имеет зависимость `depends_on: migrate (condition: service_completed_successfully)`. В виду этого миграции выполняются только один раз перед запуском реплик api. Реплики API только проверяют подключение к БД и инициализируют Redis. В эксперименте проверена обработка насыщения пула соединений, при которой клиент поулчает статус ошибки 503 вместо выдачи общего 500 для retry запросов в скрипте `python scripts/check_503_retry_after.py --n 64`. Первые запросы занимают вес пул, а оставшиеся получают response 503 с extra_headers `Retry-After`. Все запросы корректны обработаны, ключ и тело запросов не менялись. Конкуренция: 64; wall time: 0.50s; реплик наблюдалось: 2 ['api-24fde5d9', 'api-df7eae98'] результат в `results/check_overloaded_handler_503.log`. - [+] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. diff --git a/REPORT.md b/REPORT.md index 4862061c..8decf3f8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -43,6 +43,7 @@ Idempotent-Key, генерируемый при отправке запросо Полученные ID сообщений: ['41646804-dfd6-43bf-b839-983c1244c8ee', '41646804-dfd6-43bf-b839-983c1244c8ee', '41646804-dfd6-43bf-b839-983c1244c8ee', '41646804-dfd6-43bf-b839-983c1244c8ee', '41646804-dfd6-43bf-b839-983c1244c8ee'] PASS: Идемпотентность работает! Создано ровно 1 сообщение (ID: 41646804-dfd6-43bf-b839-983c1244c8ee) | results/failover_metrics.json | | Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Проверка обработки status 503 при перегрузке сервера вместо выдачи общего 500 для retry запросов| `python scripts/check_503_retry_after.py --n 64` | PASS: 503 unavailable + Retry-After correct, retries are safe, no dublicates | `results/check_overloaded_handler_503.log` | TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. diff --git a/docker-compose.yaml b/docker-compose.yaml index 90d51e3d..3b4077a9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,11 +12,20 @@ services: ports: - "8080:80" volumes: - - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro + environment: + - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx + - KEEPALIVE_DIRECTIVE=${KEEPALIVE_DIRECTIVE:-} + - PROXY_HTTP_VERSION_DIRECTIVE=${PROXY_HTTP_VERSION_DIRECTIVE:-} + - PROXY_CONNECTION_DIRECTIVE=${PROXY_CONNECTION_DIRECTIVE:-} depends_on: api: - condition: - service_healthy + condition: service_healthy + deploy: + resources: + limits: + cpus: '${NGINX_CPU:-1.0}' + memory: ${NGINX_MEM:-256M} restart: unless-stopped api: build: . @@ -27,6 +36,10 @@ services: - SESSION_TTL_SECONDS=${SESSION_TTL_SECONDS} - REDIS_URL=${REDIS_URL} - INSTANCE_ID_PREFIX=api + - DB_POOL_SIZE=${DB_POOL_SIZE:-10} + - DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20} + - DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30} + - DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-3600} depends_on: postgres: condition: service_healthy @@ -42,11 +55,11 @@ services: start_period: 15s retries: 3 deploy: - replicas: 2 + replicas: ${API_REPLICAS:-2} resources: limits: - cpus: '0.5' - memory: 512M + cpus: '${API_CPU:-1.0}' + memory: ${API_MEM:-512M} postgres: image: postgres:16.3-alpine @@ -75,6 +88,11 @@ services: - "127.0.0.1:6379:6379" volumes: - redisdata:/data + deploy: + resources: + limits: + cpus: '${REDIS_CPU:-0.5}' + memory: ${REDIS_MEM:-256M} healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s diff --git a/infra/nginx.conf.template b/infra/nginx.conf.template new file mode 100644 index 00000000..8116ae29 --- /dev/null +++ b/infra/nginx.conf.template @@ -0,0 +1,40 @@ +worker_processes auto; + +events { + worker_connections 4096; +} + +http { + access_log off; + sendfile on; + tcp_nopush on; + + upstream backend { + server api:8080 max_fails=2 fail_timeout=10s; + ${KEEPALIVE_DIRECTIVE} + } + + server { + listen 80; + server_name _; + + proxy_connect_timeout 5s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 10s; + + location / { + proxy_pass http://backend; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + ${PROXY_HTTP_VERSION_DIRECTIVE} + ${PROXY_CONNECTION_DIRECTIVE} + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9f3df1b6..20c99be5 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/scripts/check_503_retry_after.py b/scripts/check_503_retry_after.py new file mode 100644 index 00000000..e2dcb1d3 --- /dev/null +++ b/scripts/check_503_retry_after.py @@ -0,0 +1,161 @@ +import argparse +import base64 +import json +import statistics +import sys +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = "http://localhost:8080" + + +def setup(): + pwd = "SecurePassword123!" + ids = [] + for _ in range(2): + u = f"ra_{uuid.uuid4().hex[:10]}" + s, _, user, _ = call("POST", "/api/v1/users", + body={"username": u, "display_name": "RA", "password": pwd}) + assert s == 201, (s, user) + ids.append((u, user["id"])) + cred = base64.b64encode(f"{ids[0][0]}:{pwd}".encode()).decode() + s, _, sess, _ = call("POST", "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {cred}"}) + assert s == 201, (s, sess) + h = {"Authorization": f"Bearer {sess['token']}"} + s, _, chat, _ = call("POST", "/api/v1/chats", headers=h, + body={"title": "Retry-After check", "member_ids": [i[1] for i in ids]}) + assert s == 201, (s, chat) + return h, chat["id"] + + +def pct(vals, p): + if not vals: + return None + v = sorted(vals) + return round(1000 * v[min(len(v) - 1, int(p * len(v)))], 1) + + +def call(method, path, headers=None, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(BASE + path, data=data, method=method) + req.add_header("Accept", "application/json") + if data: + req.add_header("Content-Type", "application/json") + for k, v in (headers or {}).items(): + req.add_header(k, v) + t0 = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=60) as r: + raw = r.read().decode() + hdr = {k.lower(): v for k, v in r.headers.items()} + return r.status, hdr, json.loads(raw) if raw else None, time.monotonic() - t0 + except urllib.error.HTTPError as e: + raw = e.read().decode() + hdr = {k.lower(): v for k, v in e.headers.items()} + try: + parsed = json.loads(raw) if raw else None + except ValueError: + parsed = {"_raw": raw[:200]} + return e.code, hdr, parsed, time.monotonic() - t0 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=64) + ap.add_argument("--allow-zero-503", action="store_true") + args = ap.parse_args() + n = args.n + + h, chat_id = setup() + keys = [f"ra-{uuid.uuid4().hex}" for _ in range(n)] + bodies = [{"text": f"retry-after probe {i}"} for i in range(n)] + + def send(i): + s, hdr, body, dt = call("POST", f"/api/v1/chats/{chat_id}/messages", + headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i]) + return i, s, hdr, body, dt + + wall0 = time.monotonic() + with ThreadPoolExecutor(max_workers=n) as ex: + first = list(ex.map(send, range(n))) + wall = time.monotonic() - wall0 + + codes, problems, retry_idx, lat = {}, [], [], [] + instances = set() + for i, s, hdr, body, dt in first: + codes[s] = codes.get(s, 0) + 1 + lat.append(dt) + if hdr.get("x-instance-id"): + instances.add(hdr["x-instance-id"]) + if s == 503: + retry_idx.append(i) + ra = hdr.get("retry-after") + err = (body or {}).get("error", {}) + if err.get("code") != "unavailable": + problems.append(f"[{i}] code={err.get('code')}, ожидался unavailable") + if not ra or not ra.isdigit() or int(ra) < 1: + problems.append(f"[{i}] некорректный Retry-After: {ra!r}") + if err.get("request_id") != hdr.get("x-request-id"): + problems.append(f"[{i}] request_id в теле != заголовку") + if not hdr.get("x-instance-id"): + problems.append(f"[{i}] нет X-Instance-Id") + if "QueuePool" in json.dumps(body): + problems.append(f"[{i}] утечка внутренних деталей") + if s == 500: + problems.append(f"[{i}] 500 instead of 503: {body}") + + print(f"Concurrent: {n}; wall time: {wall:.2f}s; replices found: {len(instances)} {sorted(instances)}") + print(f"Statusues of 1 cycle: {codes}") + print(f"Latency, ms: p50={pct(lat,.5)} p95={pct(lat,.95)} p99={pct(lat,.99)}") + print(f"503 с Retry-After: {len(retry_idx)}") + if retry_idx: + smp = next(r for r in first if r[1] == 503) + print(f"Example: Retry-After={smp[2].get('retry-after')}, " + f"message={smp[3]['error']['message']!r}") + + ok_ids = {b["id"] for _, s, _, b, _ in first if s == 201} + for i in retry_idx: + s, hdr, body, _ = call("POST", f"/api/v1/chats/{chat_id}/messages", + headers={**h, "Idempotency-Key": keys[i]}, body=bodies[i]) + if s != 201: + problems.append(f"retry[{i}] -> {s}, expected 201: {body}") + else: + ok_ids.add(body["id"]) + + total, cursor = 0, None + while True: + p = f"/api/v1/chats/{chat_id}/messages?limit=100" + if cursor: + p += "&cursor=" + cursor + s, _, page, _ = call("GET", p, headers=h) + assert s == 200, (s, page) + total += len(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + + print(f"Messages in chat: {total}; unique id: {len(ok_ids)}; keys: {n}") + if total != n or len(ok_ids) != n: + problems.append(f"dublicates/losses: total={total}, ids={len(ok_ids)}, expected {n}") + + if problems: + print("\nFAIL:") + for p in problems[:15]: + print(" -", p) + if len(problems) > 15: + print(f" ... и ещё {len(problems)-15}") + sys.exit(1) + + if not retry_idx and not args.allow_zero_503: + print("\nINCONCLUSIVE: 503 не наблюдалось — маппинг пула НЕ проверен.") + sys.exit(2) + + print("\nPASS: 503 unavailable + Retry-After correct, retries are safe, no dublicates") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/plot_scaling.py b/scripts/plot_scaling.py new file mode 100644 index 00000000..34ab2376 --- /dev/null +++ b/scripts/plot_scaling.py @@ -0,0 +1,75 @@ +import json +import statistics +from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +RAW = ROOT / "results" / "scaling" / "raw.json" +OUT = ROOT / "results" / "scaling" +OUT.mkdir(parents=True, exist_ok=True) + +data = json.loads(RAW.read_text(encoding="utf-8")) + + +def points(cfg, key): + agg = {} + for p in cfg["points"]: + agg.setdefault(p["concurrency"], []).append(p["metrics"]) + xs = sorted(agg) + ys = [] + for x in xs: + vals = [m[key] if key in m else m["successful_latency_ms"][key] for m in agg[x]] + ys.append(statistics.median(vals)) + return xs, ys + + +def plot_series(series, fname, title): + cfgs = [c for c in data["configs"] if c["series"] == series] + if not cfgs: + return + fig, ax = plt.subplots(1, 3, figsize=(16, 4.2)) + for c in cfgs: + label = f"{c['name']} (r={c['replicas']}, Σcpu={c['replicas']*c['api_cpu']:.1f})" + x, y = points(c, "successful_rps"); ax[0].plot(x, y, "o-", label=label) + x, y = points(c, "p95"); ax[1].plot(x, y, "o-", label=label) + x, y = points(c, "error_ratio"); ax[2].plot(x, y, "o-", label=label) + ax[0].set(xlabel="concurrency", ylabel="successful RPS", title="Throughput") + ax[1].set(xlabel="concurrency", ylabel="p95 latency, ms", title="Tail latency") + ax[2].set(xlabel="concurrency", ylabel="error ratio", title="Errors") + for a in ax: + a.grid(alpha=.3); a.legend(fontsize=7) + fig.suptitle(title) + fig.tight_layout() + fig.savefig(OUT / fname, dpi=140) + print("saved", OUT / fname) + + +plot_series("A", "serie_a_fixed_budget.png", + "Serie A: fixed total API budget (2.0 CPU / 1024 MiB), 1 vs 2 vs 4 replicas") +plot_series("B", "serie_b_resource_growth.png", + "Serie B: growing API CPU, Postgres fixed") +plot_series("C", "serie_c_keepalive_ab.png", + "Serie C: nginx upstream keepalive A/B (identical resources)") +plot_series("D", "serie_d_pool_saturation.png", + "Serie D: connection-pool saturation at identical CPU (2x1.0)") + +lines = ["| config | replicas | Σcpu API | conc | median RPS | p50 | p95 | p99 | err |", + "|---|---|---|---|---|---|---|---|---|"] +for c in data["configs"]: + agg = {} + for p in c["points"]: + agg.setdefault(p["concurrency"], []).append(p["metrics"]) + for conc in sorted(agg): + ms = agg[conc] + med = lambda f: round(statistics.median([f(m) for m in ms]), 3) + lines.append( + f"| {c['name']} | {c['replicas']} | {c['replicas']*c['api_cpu']:.1f} | {conc} | " + f"{med(lambda m: m['successful_rps'])} | " + f"{med(lambda m: m['successful_latency_ms']['p50'])} | " + f"{med(lambda m: m['successful_latency_ms']['p95'])} | " + f"{med(lambda m: m['successful_latency_ms']['p99'])} | " + f"{med(lambda m: m['error_ratio'])} |") +(OUT / "summary_table.md").write_text("\n".join(lines), encoding="utf-8") +print("saved", OUT / "summary_table.md") \ No newline at end of file diff --git a/scripts/scaling_benchmark.py b/scripts/scaling_benchmark.py new file mode 100644 index 00000000..ed601f55 --- /dev/null +++ b/scripts/scaling_benchmark.py @@ -0,0 +1,278 @@ +import argparse +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RAW = ROOT / "results" / "scaling" +LOAD_DIR = RAW / "load" +STATS_DIR = RAW / "stats" +BASE_URL = os.getenv("BENCH_BASE_URL", "http://localhost:8080") + +CONCURRENCY = [8, 16, 32, 64] +DURATION = 30 +WARMUP = 10 +REPEATS = 2 + +NGINX_BASELINE = { + "KEEPALIVE_DIRECTIVE": "", + "PROXY_HTTP_VERSION_DIRECTIVE": "", + "PROXY_CONNECTION_DIRECTIVE": "", +} +NGINX_KEEPALIVE = { + "KEEPALIVE_DIRECTIVE": "keepalive 64;", + "PROXY_HTTP_VERSION_DIRECTIVE": "proxy_http_version 1.1;", + "PROXY_CONNECTION_DIRECTIVE": 'proxy_set_header Connection "";', +} + + +def series_a(): + return [ + dict(name="A_r1", replicas=1, api_cpu=2.0, api_mem="1024M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), + dict(name="A_r2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), + dict(name="A_r4", replicas=4, api_cpu=0.5, api_mem="256M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_BASELINE), + ] + + +def series_b(): + return [ + dict(name="B_cpu1", replicas=2, api_cpu=0.5, api_mem="512M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), + dict(name="B_cpu2", replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), + dict(name="B_cpu4", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), + dict(name="B_cpu4_db2", replicas=2, api_cpu=2.0, api_mem="512M", pg_cpu=2.0, + pool=10, overflow=20, pool_timeout=30, nginx=NGINX_KEEPALIVE), + ] + + +def series_c(): + base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, + pool=10, overflow=20, pool_timeout=30) + return [ + dict(name="C_before", **base, nginx=NGINX_BASELINE), + dict(name="C_after", **base, nginx=NGINX_KEEPALIVE), + ] + + +def series_d(): + base = dict(replicas=2, api_cpu=1.0, api_mem="512M", pg_cpu=1.0, + nginx=NGINX_KEEPALIVE) + return [ + dict(name="D_pool_starved", **base, pool=2, overflow=0, pool_timeout=5), + dict(name="D_pool_normal", **base, pool=10, overflow=20, pool_timeout=30), + ] + + +SERIES = {"A": series_a, "B": series_b, "C": series_c, "D": series_d} + + +def build_env(cfg): + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env.update({ + "API_REPLICAS": str(cfg["replicas"]), + "API_CPU": str(cfg["api_cpu"]), + "API_MEM": cfg["api_mem"], + "PG_CPU": str(cfg["pg_cpu"]), + "DB_POOL_SIZE": str(cfg["pool"]), + "DB_MAX_OVERFLOW": str(cfg["overflow"]), + "DB_POOL_TIMEOUT": str(cfg.get("pool_timeout", 30)), + "DB_POOL_RECYCLE": str(cfg.get("pool_recycle", 3600)), + }) + env.update(cfg["nginx"]) + return env + + +def run(cmd, env, check=True, quiet=False): + return subprocess.run( + cmd, cwd=ROOT, env=env, check=check, + stdout=subprocess.DEVNULL if quiet else None, + stderr=subprocess.STDOUT if quiet else None, + ) + + +def wait_ready(timeout=180): + deadline = time.time() + timeout + while time.time() < deadline: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r: + if r.status == 200: + return True + except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError): + pass + time.sleep(2) + return False + + +def observe_instances(n=14): + seen = set() + for _ in range(n): + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as r: + v = r.headers.get("X-Instance-Id") + if v: + seen.add(v) + except Exception: + pass + return sorted(seen) + + +def psql(sql, env): + p = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", + "-U", env.get("POSTGRES_USER", "mtusi"), + "-d", env.get("POSTGRES_DB", "mtusi-messenger"), "-At", "-c", sql], + cwd=ROOT, env=env, capture_output=True, text=True, + ) + return p.stdout.strip() if p.returncode == 0 else "" + + +def pg_snapshot(env): + out = { + "numbackends": psql("SELECT count(*) FROM pg_stat_activity WHERE datname IS NOT NULL;", env), + "active": psql("SELECT count(*) FROM pg_stat_activity WHERE state='active';", env), + "waits": psql("SELECT coalesce(wait_event_type,'cpu')||':'||count(*) FROM pg_stat_activity " + "WHERE backend_type='client backend' GROUP BY 1 ORDER BY 2 DESC;", env), + "top_sql": psql("SELECT calls||'|'||round(total_exec_time::numeric,1)||'|'||left(query,70) " + "FROM pg_stat_statements WHERE query ILIKE '%messages%' " + "AND query NOT ILIKE '%pg_stat%' ORDER BY total_exec_time DESC LIMIT 5;", env), + } + return out + + +class StatsSampler: + def __init__(self, path, interval=2.0): + self.path = path + self.interval = interval + self.proc = None + self.fh = None + + def __enter__(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + self.fh = open(self.path, "w", encoding="utf-8") + fmt = '"{{.Name}}","{{.CPUPerc}}","{{.MemUsage}}","{{.NetIO}}","{{.BlockIO}}"' + script = (f'while true; do docker stats --no-stream --format {fmt}; ' + f'echo "---SAMPLE---"; sleep {self.interval}; done') + self.proc = subprocess.Popen(["bash", "-c", script], cwd=ROOT, + stdout=self.fh, stderr=subprocess.DEVNULL) + return self + + def __exit__(self, *a): + if self.proc: + self.proc.terminate() + try: + self.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + self.proc.kill() + if self.fh: + self.fh.close() + + +def load_once(cfg, concurrency, duration, tag, env): + out = LOAD_DIR / f"{cfg['name']}_c{concurrency}_{tag}.json" + out.parent.mkdir(parents=True, exist_ok=True) + run([sys.executable, "-X", "utf8", "tests/load.py", + "--base-url", BASE_URL, + "--duration", str(duration), + "--concurrency", str(concurrency), + "--output", str(out)], env, check=False) + if not out.exists(): + return None + return json.loads(out.read_text(encoding="utf-8")) + + +def apply_config(cfg, env): + print(f"\n\nCONFIG {cfg['name']}: replicas={cfg['replicas']} " + f"cpu/replica={cfg['api_cpu']} Σcpu={cfg['replicas']*cfg['api_cpu']:.1f} " + f"pg_cpu={cfg['pg_cpu']} " + f"pool={cfg['pool']}+{cfg['overflow']} " + f"(max {cfg['replicas']*(cfg['pool']+cfg['overflow'])} conns) " + f"timeout={cfg.get('pool_timeout',30)}s\n") + + run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", + "--scale", f"api={cfg['replicas']}", "api"], env, quiet=True) + run(["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "nginx"], env, quiet=True) + if not wait_ready(): + raise RuntimeError(f"{cfg['name']}: /health/ready not reached") + time.sleep(5) + ids = observe_instances() + print(f"Observed X-Instance-Id ({len(ids)}): {ids}") + return ids + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--series", nargs="+", default=["A"], choices=list(SERIES)) + ap.add_argument("--concurrency", nargs="+", type=int, default=CONCURRENCY) + ap.add_argument("--duration", type=int, default=DURATION) + ap.add_argument("--repeats", type=int, default=REPEATS) + args = ap.parse_args() + + RAW.mkdir(parents=True, exist_ok=True) + results = { + "started_at": datetime.now(timezone.utc).isoformat(), + "base_url": BASE_URL, + "profile": {"duration_s": args.duration, "warmup_s": WARMUP, + "concurrency": args.concurrency, "repeats": args.repeats, + "tool": "tests/load.py (closed-loop, no retries)"}, + "configs": [], + } + + for s in args.series: + for cfg in SERIES[s](): + env = build_env(cfg) + ids = apply_config(cfg, env) + entry = {"series": s, **{k: v for k, v in cfg.items() if k != "nginx"}, + "nginx_mode": "keepalive" if cfg["nginx"]["KEEPALIVE_DIRECTIVE"] else "baseline", + "observed_instance_ids": ids, "points": []} + + print(f"Warmup {WARMUP}s @ c={args.concurrency[0]} ...") + load_once(cfg, args.concurrency[0], WARMUP, "warmup", env) + + for c in args.concurrency: + runs = [] + for r in range(args.repeats): + pg_before = pg_snapshot(env) + with StatsSampler(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"): + m = load_once(cfg, c, args.duration, f"r{r}", env) + pg_after = pg_snapshot(env) + if m is None: + print(f" c={c} r={r}: LOAD FAILED") + continue + runs.append(m) + print(f" c={c} r={r}: rps={m['successful_rps']} " + f"err={m['error_ratio']} p95={m['successful_latency_ms']['p95']}ms " + f"pg_backends={pg_after['numbackends']}") + entry["points"].append({ + "concurrency": c, "repeat": r, "metrics": m, + "pg_before": pg_before, "pg_after": pg_after, + "stats_csv": str(STATS_DIR / f"{cfg['name']}_c{c}_r{r}.csv"), + }) + if runs: + rps = sorted(x["successful_rps"] for x in runs) + entry.setdefault("summary", {})[str(c)] = { + "median_rps": rps[len(rps)//2], + "runs": len(runs), + } + results["configs"].append(entry) + (RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), + encoding="utf-8") + + results["finished_at"] = datetime.now(timezone.utc).isoformat() + (RAW / "raw.json").write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"\nDONE. Raw data: {RAW/'raw.json'}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/api/routes/messages/message.py b/src/api/routes/messages/message.py index 19d45771..f81a585b 100644 --- a/src/api/routes/messages/message.py +++ b/src/api/routes/messages/message.py @@ -9,7 +9,7 @@ from src.api.repositories.main_repo import Repository from src.services.idempotency import check_idempotency, body_hash, save_idempotency from src.utils import generate_id, get_now_iso from datetime import datetime - +from src.services.idempotency import release_idempotency router = APIRouter() @@ -64,19 +64,24 @@ async def send_message(chat_id: UUID, msg_in: CreateMessage, session: DbSession, if is_cached: return cached_response - message = await repo.messages.create(chat_id=chat_id, + try: + message = await repo.messages.create(chat_id=chat_id, sender_id=current_user.id, text=msg_in.text) - response_data = Message( - id=str(message.id), - chat_id=str(message.chat_id), - sender_id=str(message.sender_id), - text=message.text, - created_at=message.created_at.strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" - ) + response_data = Message( + id=str(message.id), + chat_id=str(message.chat_id), + sender_id=str(message.sender_id), + text=message.text, + created_at=message.created_at.strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + ) - response_dict = response_data.model_dump() + response_dict = response_data.model_dump() + + except Exception: + await release_idempotency(str(current_user.id), str(chat_id), idempotency_key) + raise await save_idempotency( str(current_user.id), diff --git a/src/config.py b/src/config.py index fbcd5182..9ef93c74 100644 --- a/src/config.py +++ b/src/config.py @@ -13,9 +13,12 @@ class Settings(BaseSettings): DB_POOL_SIZE: int = 10 DB_MAX_OVERFLOW: int = 20 - DB_POOL_TIMEOUT: int = 30 + DB_POOL_TIMEOUT: int = 30.0 DB_POOL_RECYCLE: int = 3600 + RETRY_AFTER_POOL: int = 1 + RETRY_AFTER_DB: int = 5 + model_config = SettingsConfigDict(env_file=".env", case_sensitive=False, extra="ignore") diff --git a/src/database.py b/src/database.py index c1897249..4aee91ae 100644 --- a/src/database.py +++ b/src/database.py @@ -17,6 +17,14 @@ engine = create_async_engine( pool_pre_ping=True, ) +print( + f"[db] pool_size={settings.DB_POOL_SIZE} " + f"max_overflow={settings.DB_MAX_OVERFLOW} " + f"max_total_per_replica={settings.DB_POOL_SIZE + settings.DB_MAX_OVERFLOW} " + f"pool_timeout={settings.DB_POOL_TIMEOUT}s " + f"pool_recycle={settings.DB_POOL_RECYCLE}s", + flush=True, +) AsyncSessionLocal = async_sessionmaker( engine, diff --git a/src/main.py b/src/main.py index 51642555..40928cc0 100644 --- a/src/main.py +++ b/src/main.py @@ -11,7 +11,9 @@ from src.api.routes.messages import message from src.api.routes.auth import auth from src.lifespan import lifespan from sqlalchemy.exc import OperationalError as SAOperationalError +from sqlalchemy.exc import TimeoutError as SATimeoutError from redis.exceptions import RedisError +from src.config import settings INSTANCE_ID = os.getenv("INSTANCE_ID") or f"api-{uuid.uuid4().hex[:8]}" @@ -51,6 +53,20 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, ) + +@application.exception_handler(SATimeoutError) +async def pool_exhausted_handler(request: Request, exc: SATimeoutError): + request_id = getattr(request.state, "request_id", None) or str(generate_id()) + return create_error_response( + message="service temporarily overloaded, retry later", + code="unavailable", + status_code=503, + request_id=request_id, + instance_id=INSTANCE_ID, + extra_headers={"Retry-After": str(settings.RETRY_AFTER_POOL)}, + ) + + @application.exception_handler(SAOperationalError) async def db_unavailable_handler(request: Request, exc: SAOperationalError): request_id = getattr(request.state, "request_id", str(generate_id())) @@ -103,6 +119,13 @@ async def global_exception_handler(request: Request, exc: Exception): if request_id is None: request_id = str(generate_id()) print(f'Internal Error: {exc}') + + print( + f"[error] request_id={request_id} " + f"type={type(exc).__module__}.{type(exc).__qualname__} " + f"msg={exc}", + flush=True, + ) return create_error_response(message="Internal Server Error", code="internal_error", status_code=500, request_id=request_id, instance_id=INSTANCE_ID) diff --git a/src/services/idempotency.py b/src/services/idempotency.py index 4010b39c..91c2791e 100644 --- a/src/services/idempotency.py +++ b/src/services/idempotency.py @@ -1,13 +1,16 @@ import hashlib import json import asyncio +import time from redis.exceptions import RedisError from src.redis_client import get_redis from fastapi import HTTPException, status +from src.config import settings IDEMPOTENCY_PREFIX = "idempotency:" IDEMPOTENCY_TTL = 86400 LOCK_TIMEOUT = 10 +CONCURRENT_WAIT_S = 2.0 def _cache_key(user_id: str, chat_id: str, idempotency_key: str) -> str: @@ -23,6 +26,31 @@ def body_hash(body: dict) -> str: return hashlib.sha256(canonical.encode("utf-8")).hexdigest() +def _conflict() -> HTTPException: + return HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Idempotency-Key already used with different body", + ) + + +def _in_progress() -> HTTPException: + return HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="concurrent request with the same Idempotency-Key is in progress", + headers={"Retry-After": str(settings.RETRY_AFTER_POOL)}, + ) + + +async def _read_result(r, cache_key: str, current_hash: str): + raw = await r.get(cache_key) + if not raw: + return None + cached = json.loads(raw) + if cached["body_hash"] != current_hash: + raise _conflict() + return cached["response"] + + async def check_idempotency(user_id: str, chat_id: str, idempotency_key: str, current_hash: str) -> dict | None: r = get_redis() if not r: @@ -31,36 +59,37 @@ async def check_idempotency(user_id: str, chat_id: str, idempotency_key: str, cu cache_key = _cache_key(user_id, chat_id, idempotency_key) lock_key = _lock_key(user_id, chat_id, idempotency_key) - raw = await r.get(cache_key) - if raw: - cached = json.loads(raw) - if cached["body_hash"] != current_hash: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Idempotency-Key already used with different body") - return True, cached["response"] + try: + result = await _read_result(r, cache_key, current_hash) + if result is not None: + return True, result - lock_acquired = await r.set(lock_key, "1", nx=True, ex=LOCK_TIMEOUT) + lock_acquired = await r.set(lock_key, "1", nx=True, ex=LOCK_TIMEOUT) + + if not lock_acquired: + deadline = time.monotonic() + CONCURRENT_WAIT_S + delay = 0.02 + while time.monotonic() < deadline: + await asyncio.sleep(delay) + result = await _read_result(r, cache_key, current_hash) + if result is not None: + return True, result + delay = min(delay * 2, 0.2) + raise _in_progress() - if not lock_acquired: - await asyncio.sleep(0.1) raw = await r.get(cache_key) if raw: cached = json.loads(raw) if cached["body_hash"] != current_hash: + await r.delete(lock_key) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Idempotency-Key already used with different body") + await r.delete(lock_key) return True, cached["response"] - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Concurrent request with same key is being processed") - - raw = await r.get(cache_key) - if raw: - cached = json.loads(raw) - if cached["body_hash"] != current_hash: - await r.delete(lock_key) - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Idempotency-Key already used with different body") - await r.delete(lock_key) - return True, cached["response"] - - return False, None + return False, None + + except RedisError: + return False, None async def save_idempotency(user_id: str, chat_id: str, idempotency_key: str, current_hash: str, response_data: dict) -> None: @@ -79,3 +108,14 @@ async def save_idempotency(user_id: str, chat_id: str, idempotency_key: str, cur ) finally: await r.delete(lock_key) + + +async def release_idempotency(user_id: str, chat_id: str, idempotency_key: str) -> None: + r = get_redis() + if r is None: + return + try: + await r.delete(_lock_key(user_id, chat_id, idempotency_key)) + except RedisError: + pass + \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index 32481539..07d4e50b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -11,6 +11,11 @@ def get_now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def create_error_response(message: str, code: str, status_code: int, request_id: str, instance_id: str) -> JSONResponse: +def create_error_response(message: str, code: str, status_code: int, request_id: str, instance_id: str, extra_headers: dict[str, str] | None = None) -> JSONResponse: body = {"error": {"code": code, "message": message, "request_id": request_id}} - return JSONResponse(status_code=status_code, content=body, headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}) + headers = {"X-Request-Id": request_id, "X-Instance-Id": instance_id} + + if extra_headers: + headers.update(extra_headers) + + return JSONResponse(status_code=status_code, content=body, headers=headers)