forked from mtusi2026/mtusi2026lab3
feat: HAProxy и N API-реплик, X-Instance-Id, общая идемпотентность, drain при SIGTERM
- compose: один масштабируемый сервис api (deploy.replicas=2, без ports и container_name), HAProxy 3.2 LTS по digest — единственная точка входа (127.0.0.1), разовый migrate до старта реплик, CPU/RAM-лимиты у всех сервисов, каждому сервису только его secrets. - HAProxy: server-template + Docker DNS (re-resolve 1 с), активная проверка GET /health/ready каждую 1 с (fall 2 / rise 2), без backup-серверов, retry только conn-failure, ошибки самого LB — в формате контракта с X-Request-Id. - API: X-Instance-Id = hostname контейнера на каждом ответе (и на 500/503); Idempotency-Key обязателен; ключ, сообщение и результат пишутся в ОДНОЙ транзакции (миграция 0002), конкурентный повтор ждёт уникальный индекс; другой body — 409, посторонний — всегда 404; attachment_ids=[] равен отсутствию поля; дубли ключей JSON и тело у POST /auth/sessions — 400. - SIGTERM: readiness 503 + DRAIN_SECONDS=3 обслуживания, затем graceful stop. - scripts: stop-one.sh (+ .cmd для Windows), restore-api.sh, restart.sh перезапускает и LB, lbctl.py — состояние бэкендов HAProxy -> контейнеры. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4f6cfda37a
commit
3f12698676
+17
-1
@@ -11,6 +11,10 @@ DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`.
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas).
|
||||
API_REPLICAS=2
|
||||
# Idempotency-Key results are kept at least this long (contract: >= 24 h).
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Additional settings of this implementation (defaults shown; all optional):
|
||||
LOG_LEVEL=info
|
||||
DB_POOL_MIN=1
|
||||
@@ -19,5 +23,17 @@ DB_TIMEOUT_SECONDS=3
|
||||
CACHE_TIMEOUT_SECONDS=0.5
|
||||
PASSWORD_HASH_CONCURRENCY=2
|
||||
DOCS_ENABLED=true
|
||||
# Host port of the API (compose publishes it on 127.0.0.1 only)
|
||||
# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests.
|
||||
DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=8
|
||||
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=256m
|
||||
PG_CPUS=2.0
|
||||
PG_MEMORY=1g
|
||||
VALKEY_CPUS=0.5
|
||||
LB_CPUS=1.0
|
||||
PG_MAX_CONNECTIONS=100
|
||||
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
|
||||
APP_PUBLISH_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Messenger API, lab 2 (PostgreSQL + Valkey). Built by compose.yaml (services api, migrate).
|
||||
# Messenger API, lab 3 (replicas behind HAProxy, PostgreSQL + Valkey). Built by compose.yaml (services api, migrate).
|
||||
#
|
||||
# Python is interpreted, so there is no separate compile stage; the image
|
||||
# contains only the interpreter, locked dependencies and the source code.
|
||||
|
||||
@@ -30,14 +30,14 @@ test-failover:
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Student solution (lab 2, Python). On Windows without make run the commands from
|
||||
# REPORT.md directly (public tests there need PYTHONUTF8=1; restart hook: scripts/restart.cmd).
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain
|
||||
# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
|
||||
# Build, apply migrations (service migrate) and wait until every service is healthy.
|
||||
# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy.
|
||||
up: secrets
|
||||
docker compose up -d --build --wait
|
||||
|
||||
@@ -46,14 +46,32 @@ down:
|
||||
docker compose down
|
||||
|
||||
logs:
|
||||
docker compose logs --tail 50 api
|
||||
docker compose logs --tail 50 api lb
|
||||
|
||||
ps:
|
||||
docker compose ps
|
||||
docker compose ps -a
|
||||
|
||||
migrate:
|
||||
docker compose run --rm migrate
|
||||
|
||||
# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id.
|
||||
lb-status:
|
||||
$(PYTHON) scripts/lbctl.py status
|
||||
$(PYTHON) scripts/lbctl.py replicas
|
||||
|
||||
# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP.
|
||||
restore:
|
||||
sh scripts/restore-api.sh
|
||||
$(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2}
|
||||
|
||||
# Replace every API replica one by one with a freshly built image (no downtime).
|
||||
rolling-update:
|
||||
sh scripts/rolling-update.sh
|
||||
|
||||
# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
|
||||
failover-series:
|
||||
$(PYTHON) bench/failover_series.py --output evidence/failover
|
||||
|
||||
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
|
||||
student-test:
|
||||
$(PYTHON) -m pytest student-tests -v
|
||||
|
||||
+62
-10
@@ -1,9 +1,12 @@
|
||||
# Lab 2 stand: API + PostgreSQL + Valkey. Start from a clean clone:
|
||||
# Lab 3 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) -> PostgreSQL + Valkey.
|
||||
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
|
||||
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
|
||||
# cp .env.example .env # optional overrides
|
||||
# docker compose up -d --build --wait
|
||||
# Images are pinned by version AND digest. Passwords are passed only as secret files.
|
||||
name: mtusi-lab2
|
||||
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
|
||||
# docker compose --env-file .env up -d --build --wait
|
||||
# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb.
|
||||
# Images are pinned by version AND digest. Passwords are passed only as secret files, and
|
||||
# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only).
|
||||
name: mtusi-lab3
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
@@ -17,6 +20,10 @@ x-api-env: &api-env
|
||||
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
|
||||
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
# Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests.
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
|
||||
DOCS_ENABLED: ${DOCS_ENABLED:-true}
|
||||
|
||||
services:
|
||||
@@ -26,6 +33,8 @@ services:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
# pg_stat_statements: per-statement counters for the track 5 experiments (bench/track1).
|
||||
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path
|
||||
@@ -37,6 +46,9 @@ services:
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
@@ -61,10 +73,15 @@ services:
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
|
||||
|
||||
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
|
||||
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab2
|
||||
image: mtusi-messenger:lab3
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
@@ -75,14 +92,24 @@ services:
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi-messenger:lab2
|
||||
image: mtusi-messenger:lab3
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url]
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # only on localhost of the host
|
||||
# No `ports` and no container_name: replicas are reachable only through `lb`,
|
||||
# and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname.
|
||||
expose: ["8080"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -90,8 +117,33 @@ services:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 10s
|
||||
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-256m}"}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # the ONLY public entry point
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas)
|
||||
healthcheck:
|
||||
# Config is valid and the process answers; whether any API is ready is the LB's own job.
|
||||
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# HAProxy 3.2 LTS — the single public entry point of the lab 3 stand (compose service `lb`).
|
||||
#
|
||||
# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the
|
||||
# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and
|
||||
# re-resolves every second, so scale up/down, a restarted container with a new IP and a
|
||||
# full stack restart are picked up without restarting the LB. A stopped container drops
|
||||
# out of DNS and its slot goes to MAINT.
|
||||
#
|
||||
# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey reachable, not draining)
|
||||
# every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide anything on
|
||||
# their own, and there are NO backup/fallback servers without checks: a replica that answers
|
||||
# /health/live but not /health/ready never gets business traffic.
|
||||
# Worst case detection of a failed replica:
|
||||
# inter 1 s + fall 2 x (check timeout 1 s) ~= 3 s (frozen process);
|
||||
# refused connection / 503 ready: inter 1 s + fastinter 0.5 s ~= 1.5 s.
|
||||
# Requests that could not even be SENT (connection refused/reset before the request)
|
||||
# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST,
|
||||
# because the API has not seen them. Nothing that may have reached the API is retried by
|
||||
# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key.
|
||||
# No sticky sessions: sessions live in Valkey, every replica serves every user.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
# Runtime API for `show servers state` / `show stat` (scripts and tests, no network port).
|
||||
stats socket /var/lib/haproxy/admin.sock mode 600 level admin
|
||||
hard-stop-after 15s
|
||||
|
||||
resolvers docker
|
||||
nameserver dns1 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 1s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
option http-keep-alive
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s
|
||||
timeout queue 3s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
# Errors generated by the LB itself, in the contract error format with the same request id.
|
||||
# They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the
|
||||
# request is unknown (a replica died mid-request / did not answer in time): retry with the
|
||||
# same Idempotency-Key. 503 = no ready replica at all.
|
||||
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# One request id end to end: keep a valid client X-Request-Id, otherwise create one; the
|
||||
# API echoes it, and errors generated by the LB itself carry the same value.
|
||||
http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ }
|
||||
http-request set-var(txn.request_id) req.hdr(x-request-id)
|
||||
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\""
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
|
||||
server-template api 8 api:8080
|
||||
|
||||
# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now.
|
||||
frontend stats
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1,25 @@
|
||||
-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages.
|
||||
-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe
|
||||
-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции").
|
||||
--
|
||||
-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key.
|
||||
-- The method (POST) and the path template are fixed for this table, so the scope
|
||||
-- is (user_id, chat_id, key). The key row, the message and the result are written in
|
||||
-- ONE transaction (repo.post_message): there is no "pending" state that a crash could
|
||||
-- leave behind, and the stored result is the message row itself.
|
||||
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
key text NOT NULL,
|
||||
request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body
|
||||
-- Inserted before the message in the same transaction, hence checked at COMMIT.
|
||||
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, chat_id, key),
|
||||
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_keys_message_key UNIQUE (message_id)
|
||||
);
|
||||
-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h).
|
||||
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""What the load balancer thinks right now, mapped to real containers.
|
||||
|
||||
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
|
||||
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
|
||||
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
|
||||
|
||||
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
|
||||
so no admin port is published. Container hostname = X-Instance-Id of that replica.
|
||||
Prints no secrets. Used by student-tests and bench scripts as a library as well.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
|
||||
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
|
||||
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
|
||||
|
||||
|
||||
def run(*args: str, timeout: float = 60) -> str:
|
||||
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
timeout=timeout)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def replicas(all_states: bool = True) -> list[dict]:
|
||||
"""API containers of this compose project (running and stopped)."""
|
||||
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
|
||||
if not ids:
|
||||
return []
|
||||
data = json.loads(run("docker", "inspect", *ids))
|
||||
result = []
|
||||
for item in data:
|
||||
networks = item["NetworkSettings"]["Networks"] or {}
|
||||
result.append({
|
||||
"id": item["Id"][:12],
|
||||
"name": item["Name"].lstrip("/"),
|
||||
"hostname": item["Config"]["Hostname"],
|
||||
"state": item["State"]["Status"],
|
||||
"health": (item["State"].get("Health") or {}).get("Status", "-"),
|
||||
"exit_code": item["State"]["ExitCode"],
|
||||
"started_at": item["State"]["StartedAt"],
|
||||
"pid": item["State"]["Pid"],
|
||||
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
|
||||
})
|
||||
return sorted(result, key=lambda r: r["name"])
|
||||
|
||||
|
||||
def lb_servers() -> list[dict]:
|
||||
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
|
||||
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
|
||||
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
|
||||
servers = []
|
||||
for line in raw.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 7 or not parts[0].isdigit():
|
||||
continue
|
||||
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
|
||||
if admin & ADMIN_RMAINT or addr == "-":
|
||||
state = "MAINT"
|
||||
else:
|
||||
state = OP_STATE.get(op, str(op))
|
||||
container = by_ip.get(addr)
|
||||
servers.append({"slot": name, "addr": addr, "state": state,
|
||||
"container": container["name"] if container else None,
|
||||
"instance_id": container["hostname"] if container else None})
|
||||
return servers
|
||||
|
||||
|
||||
def up_instances() -> set[str]:
|
||||
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
|
||||
|
||||
|
||||
def wait_up(count: int, timeout: float = 60) -> float:
|
||||
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
if len(up_instances()) == count:
|
||||
return time.monotonic() - started
|
||||
except RuntimeError:
|
||||
pass # lb restarting
|
||||
if time.monotonic() - started > timeout:
|
||||
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("status")
|
||||
sub.add_parser("replicas")
|
||||
wait = sub.add_parser("wait-up")
|
||||
wait.add_argument("count", type=int)
|
||||
wait.add_argument("--timeout", type=float, default=60)
|
||||
args = parser.parse_args()
|
||||
if args.command == "status":
|
||||
for s in lb_servers():
|
||||
if s["state"] != "MAINT" or s["container"]:
|
||||
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
|
||||
print(f"(other slots: MAINT, no container)")
|
||||
elif args.command == "replicas":
|
||||
for r in replicas():
|
||||
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
|
||||
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
|
||||
else:
|
||||
took = wait_up(args.count, args.timeout)
|
||||
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+7
-5
@@ -1,15 +1,17 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
# Restarts the API and ALL state stores (PostgreSQL, Valkey) of compose.yaml.
|
||||
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL and Valkey.
|
||||
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
|
||||
# Output is suppressed by the test; run it by hand to see the steps.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# 1. API first: no request is served while the stores restart underneath it.
|
||||
docker compose stop api
|
||||
# 1. Entry point and API first: no request is served while the stores restart underneath.
|
||||
docker compose stop lb api
|
||||
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again.
|
||||
docker compose restart postgres valkey
|
||||
# 3. API again; it waits for healthy stores via depends_on and reconnects its pool.
|
||||
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
|
||||
# the new replica addresses through Docker DNS by itself - no stale backend list.
|
||||
docker compose start api
|
||||
echo "restart.sh: api, postgres and valkey restarted with volumes preserved"
|
||||
docker compose start lb
|
||||
echo "restart.sh: lb, api (all replicas), postgres and valkey restarted with volumes preserved"
|
||||
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
|
||||
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
|
||||
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-recreate --no-deps --wait api
|
||||
docker compose ps api --format '{{.Name}} {{.Status}}'
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
|
||||
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
|
||||
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
|
||||
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
|
||||
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
|
||||
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
|
||||
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
|
||||
# restore it explicitly afterwards: scripts/restore-api.sh.
|
||||
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
|
||||
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
|
||||
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
mode="${STOP_MODE:-stop}"
|
||||
|
||||
target=""
|
||||
# Only RUNNING containers of THIS compose project's service `api` are candidates.
|
||||
for id in $(docker compose ps -q api); do
|
||||
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target="$id"
|
||||
fi
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
|
||||
exit 3
|
||||
fi
|
||||
name="$(docker inspect -f '{{.Name}}' "$target")"
|
||||
case "$mode" in
|
||||
stop) docker stop --time 15 "$target" >/dev/null ;;
|
||||
kill) docker stop --time 0 "$target" >/dev/null ;;
|
||||
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
|
||||
esac
|
||||
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"
|
||||
+32
-10
@@ -1,16 +1,20 @@
|
||||
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
|
||||
|
||||
Shutdown sequence on SIGTERM/SIGINT:
|
||||
1. readiness flips to 503 (`app.state.draining = True`);
|
||||
2. uvicorn closes the listening socket, so new connections are refused;
|
||||
3. in-flight requests get up to SHUTDOWN_TIMEOUT_SECONDS (default 8 s < Docker's 10 s) to finish;
|
||||
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
|
||||
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
|
||||
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
|
||||
marked this replica DOWN and sends nothing new here;
|
||||
3. uvicorn closes the listening socket; in-flight requests get up to
|
||||
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
|
||||
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
|
||||
Data stays in PostgreSQL and Valkey volumes.
|
||||
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
|
||||
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
from types import FrameType
|
||||
|
||||
import uvicorn
|
||||
@@ -23,12 +27,30 @@ log = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class GracefulServer(uvicorn.Server):
|
||||
"""uvicorn.Server that marks the app as draining before the normal graceful stop."""
|
||||
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
|
||||
|
||||
def __init__(self, config: uvicorn.Config, drain_seconds: float):
|
||||
super().__init__(config)
|
||||
self.drain_seconds = drain_seconds
|
||||
self.drain_timer: threading.Timer | None = None
|
||||
|
||||
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
|
||||
if not self.should_exit:
|
||||
log.info("shutdown_requested", extra={"fields": {"signal": sig, "timeout_s": self.config.timeout_graceful_shutdown}})
|
||||
self.config.app.state.draining = True # config.app is our Starlette instance
|
||||
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
|
||||
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
|
||||
"timeout_s": self.config.timeout_graceful_shutdown}})
|
||||
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
|
||||
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
|
||||
self.drain_timer.daemon = True
|
||||
self.drain_timer.start()
|
||||
return
|
||||
if self.drain_timer is not None:
|
||||
self.drain_timer.cancel() # second signal: stop draining now
|
||||
self._stop(sig, frame)
|
||||
|
||||
def _stop(self, sig: int, frame: FrameType | None) -> None:
|
||||
if not self.should_exit:
|
||||
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
@@ -51,10 +73,10 @@ def main() -> int:
|
||||
lifespan="on",
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
)
|
||||
server = GracefulServer(config)
|
||||
server = GracefulServer(config, settings.drain_seconds)
|
||||
log.info("starting", extra={"fields": {
|
||||
"version": __version__, "env": settings.env, "host": settings.host, "port": settings.port,
|
||||
"python": sys.version.split()[0], "storage": "postgresql",
|
||||
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
|
||||
}})
|
||||
server.run()
|
||||
log.info("stopped")
|
||||
|
||||
+65
-12
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
@@ -19,7 +21,7 @@ from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Sen
|
||||
from . import authhdr, cursor, docs, passwords, repo, validation
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, PayloadTooLarge, Unauthorized, Unavailable
|
||||
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
|
||||
from .sessions import SessionStore, build_store
|
||||
from .timefmt import rfc3339
|
||||
|
||||
@@ -56,10 +58,12 @@ async def http_exception_handler(request: Request, exc: Exception) -> JSONRespon
|
||||
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
|
||||
# --------------------------------------------------------------------------
|
||||
class RequestContextMiddleware:
|
||||
"""Assigns X-Request-Id to every response, writes one access-log line, turns crashes into 500."""
|
||||
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
|
||||
included), writes one access-log line, turns crashes into 500."""
|
||||
|
||||
def __init__(self, app: ASGIApp):
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id.encode("ascii")
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
@@ -77,8 +81,10 @@ class RequestContextMiddleware:
|
||||
if message["type"] == "http.response.start":
|
||||
started = True
|
||||
status = message["status"]
|
||||
headers = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"]
|
||||
headers = [(k, v) for k, v in message.get("headers", [])
|
||||
if k.lower() not in (b"x-request-id", b"x-instance-id")]
|
||||
headers.append((b"x-request-id", request_id.encode("latin-1")))
|
||||
headers.append((b"x-instance-id", self.instance_id))
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
@@ -90,20 +96,21 @@ class RequestContextMiddleware:
|
||||
if not started:
|
||||
status = 503
|
||||
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
|
||||
await response(scope, receive, send)
|
||||
await response(scope, receive, send_with_id)
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
|
||||
if not started:
|
||||
status = 500
|
||||
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
await response(scope, receive, send)
|
||||
await response(scope, receive, send_with_id)
|
||||
finally:
|
||||
access_log.info("request", extra={"fields": {
|
||||
"request_id": request_id,
|
||||
"method": scope["method"],
|
||||
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
|
||||
"status": status,
|
||||
"instance_id": self.instance_id.decode("ascii"),
|
||||
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
|
||||
}})
|
||||
|
||||
@@ -166,6 +173,27 @@ def page_params(request: Request) -> tuple[int, str | None]:
|
||||
return limit, raw_cursor
|
||||
|
||||
|
||||
async def reject_body(request: Request) -> None:
|
||||
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
|
||||
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
|
||||
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
||||
async for chunk in request.stream():
|
||||
if chunk:
|
||||
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
||||
|
||||
|
||||
def idempotency_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
return validation.idempotency_key(values[0] if len(values) == 1 else None)
|
||||
|
||||
|
||||
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
|
||||
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
|
||||
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
|
||||
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def ok(body: Any, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(body, status_code=status)
|
||||
|
||||
@@ -205,6 +233,7 @@ async def register_user(request: Request) -> JSONResponse:
|
||||
async def create_session(request: Request) -> JSONResponse:
|
||||
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
|
||||
username, password = authhdr.parse_basic(request.headers.get("authorization"))
|
||||
await reject_body(request)
|
||||
found = await repo.get_credentials(db_of(request), username)
|
||||
user, stored_hash = found if found else (None, None)
|
||||
if not await passwords.verify_password(stored_hash, password) or user is None:
|
||||
@@ -262,13 +291,23 @@ async def create_message(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
db = db_of(request)
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before the body is read
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
|
||||
key = idempotency_key(request)
|
||||
data = await read_json(request)
|
||||
validation.check_fields(data, {"text"})
|
||||
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
|
||||
text = validation.text_field(data, "text", 4000)
|
||||
attachment_ids = validation.attachment_ids_field(data)
|
||||
if attachment_ids:
|
||||
# Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404).
|
||||
raise NotFound("Вложение не найдено")
|
||||
# sender_id is derived from the session, never taken from the body.
|
||||
message = await repo.add_message(db, user_id, chat_id, text)
|
||||
return ok(message.to_json(), 201)
|
||||
settings = request.app.state.settings
|
||||
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
|
||||
settings.idempotency_ttl_seconds)
|
||||
if result.replayed:
|
||||
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
|
||||
"message_id": result.message.id}})
|
||||
return ok(result.message.to_json(), 201)
|
||||
|
||||
|
||||
async def list_messages(request: Request) -> JSONResponse:
|
||||
@@ -288,16 +327,30 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
db = Database(settings)
|
||||
sessions = build_store(settings, db)
|
||||
|
||||
async def purge_loop() -> None:
|
||||
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
|
||||
while True:
|
||||
await asyncio.sleep(600)
|
||||
try:
|
||||
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
|
||||
if deleted:
|
||||
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
|
||||
except ApiError:
|
||||
pass # database unavailable right now; next round
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def lifespan(app: Starlette):
|
||||
passwords.configure(settings.hash_concurrency)
|
||||
await db.open()
|
||||
log.info("dependencies_configured", extra={"fields": {
|
||||
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds}})
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
||||
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds}})
|
||||
purge = asyncio.create_task(purge_loop())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
purge.cancel()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
|
||||
@@ -324,7 +377,7 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
|
||||
app.state.settings = settings
|
||||
app.state.db = db
|
||||
app.state.sessions = sessions
|
||||
|
||||
+29
-2
@@ -8,6 +8,8 @@ Invalid values stop the process at startup with a clear message.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
@@ -16,6 +18,22 @@ class ConfigError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||||
|
||||
|
||||
def _instance_id() -> str:
|
||||
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
|
||||
|
||||
Docker sets the hostname of a container to its short container ID, so the value
|
||||
is unique per replica, stable for the life of the process and maps 1:1 to a real
|
||||
container (`docker inspect`), which scripts/stop-one.sh relies on.
|
||||
"""
|
||||
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
||||
if INSTANCE_ID_RE.fullmatch(value) is None:
|
||||
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
|
||||
return value
|
||||
|
||||
|
||||
def _int(name: str, default: int, low: int, high: int) -> int:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
@@ -85,6 +103,9 @@ class Settings:
|
||||
db_timeout_seconds: float = 3.0
|
||||
cache_timeout_seconds: float = 0.5
|
||||
hash_concurrency: int = 2
|
||||
instance_id: str = "local"
|
||||
drain_seconds: float = 0.0
|
||||
idempotency_ttl_seconds: int = 86400
|
||||
|
||||
@property
|
||||
def db_statement_timeout_ms(self) -> int:
|
||||
@@ -110,8 +131,8 @@ class Settings:
|
||||
port=_int("APP_PORT", 8080, 1, 65535),
|
||||
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
|
||||
log_level=log_level,
|
||||
# Docker sends SIGKILL 10 s after SIGTERM by default; finish earlier.
|
||||
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 9.0),
|
||||
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
|
||||
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
|
||||
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
|
||||
docs_enabled=_bool("DOCS_ENABLED", True),
|
||||
database_url=_secret("DATABASE_URL", required=True),
|
||||
@@ -124,4 +145,10 @@ class Settings:
|
||||
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
|
||||
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
|
||||
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
|
||||
instance_id=_instance_id(),
|
||||
# After SIGTERM: readiness is 503 but requests are still served for this long,
|
||||
# so the load balancer stops routing here before the listener closes.
|
||||
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
|
||||
# Contract: a replay within 24 h returns the original result.
|
||||
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
|
||||
)
|
||||
|
||||
+74
-13
@@ -15,7 +15,7 @@ from psycopg import errors as pg_errors
|
||||
|
||||
from .cursor import SortKey
|
||||
from .db import Database
|
||||
from .errors import Conflict, InvalidRequest, NotFound
|
||||
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
|
||||
from .timefmt import from_us, rfc3339, to_us
|
||||
|
||||
|
||||
@@ -198,28 +198,89 @@ async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: i
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ messages
|
||||
async def add_message(db: Database, user_id: str, chat_id: str, text: str) -> Message:
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class PostResult:
|
||||
message: Message
|
||||
replayed: bool # True: the key was already used with this body, nothing new was written
|
||||
|
||||
|
||||
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
|
||||
text: str) -> dt.datetime:
|
||||
"""Insert with a per-chat strictly increasing timestamp.
|
||||
|
||||
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
|
||||
chat are serialised: a message with a smaller created_at can never commit after
|
||||
a message with a bigger one, and a reader paging by cursor cannot skip it.
|
||||
"""
|
||||
cur = await conn.execute(
|
||||
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
|
||||
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
|
||||
(chat_id,))
|
||||
(created_at,) = await cur.fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_id, user_id, _b(text), created_at))
|
||||
return created_at
|
||||
|
||||
|
||||
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
|
||||
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
|
||||
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
|
||||
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
|
||||
CLAIM_KEY = (
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
|
||||
"VALUES (%s, %s, %s, %s, %s) "
|
||||
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
|
||||
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
|
||||
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"RETURNING 1")
|
||||
STORED_RESULT = (
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k "
|
||||
"JOIN messages m ON m.id = k.message_id "
|
||||
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
|
||||
|
||||
|
||||
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
|
||||
ttl_seconds: int) -> PostResult:
|
||||
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
|
||||
|
||||
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
|
||||
ever left behind: a crash or 5xx rolls everything back and the client may retry);
|
||||
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
|
||||
* same key, other body -> 409; the membership check comes first, so outsiders always get 404.
|
||||
"""
|
||||
message_id = uuid.uuid4()
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
|
||||
async def work(conn: psycopg.AsyncConnection) -> PostResult:
|
||||
await _check_member(conn, user_id, chat_id)
|
||||
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
|
||||
if await cur.fetchone() is not None:
|
||||
created_at = await _insert_message(conn, message_id, user_id, chat_id, text)
|
||||
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False)
|
||||
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
# The key row vanished between the two statements (expired-key cleanup): retryable.
|
||||
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
||||
if bytes(row[0]) != request_hash:
|
||||
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
||||
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True)
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
|
||||
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> int:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
|
||||
if not (await cur.fetchone())[0]:
|
||||
return 0
|
||||
cur = await conn.execute(
|
||||
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
|
||||
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
|
||||
(chat_id,))
|
||||
(created_at,) = await cur.fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_id, user_id, _b(text), created_at))
|
||||
return created_at
|
||||
created_at = await db.run(work)
|
||||
return Message(str(message_id), chat_id, user_id, text, created_at)
|
||||
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
|
||||
"SELECT ctid FROM idempotency_keys "
|
||||
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
|
||||
(ttl_seconds, batch))
|
||||
return cur.rowcount
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
|
||||
|
||||
@@ -16,6 +16,8 @@ USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
|
||||
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
LIMIT_RE = re.compile(r"[0-9]{1,3}")
|
||||
# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
SURROGATES_RE = re.compile("[\ud800-\udfff]")
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
@@ -27,11 +29,19 @@ def _reject_constant(name: str) -> Any:
|
||||
raise ValueError(f"invalid JSON constant {name}")
|
||||
|
||||
|
||||
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
# {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins".
|
||||
result = dict(pairs)
|
||||
if len(result) != len(pairs):
|
||||
raise ValueError("duplicate key in JSON object")
|
||||
return result
|
||||
|
||||
|
||||
def parse_json_object(body: bytes) -> dict[str, Any]:
|
||||
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
value = json.loads(text, parse_constant=_reject_constant)
|
||||
value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys)
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
|
||||
if not isinstance(value, dict):
|
||||
@@ -102,6 +112,26 @@ def member_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
return ids
|
||||
|
||||
|
||||
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
"""Optional `attachment_ids` (lab 4+). Absent and [] are the same message (contracts/README.md);
|
||||
lab 3 has no attachments yet, so a non-empty list can only reference unknown objects."""
|
||||
value = data.get("attachment_ids", [])
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest("attachment_ids должен быть массивом")
|
||||
if len(value) > 10:
|
||||
raise InvalidRequest("attachment_ids содержит больше 10 элементов")
|
||||
ids = [parse_uuid(item, "UUID вложения") for item in value]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise InvalidRequest("attachment_ids содержит дубликаты")
|
||||
return ids
|
||||
|
||||
|
||||
def idempotency_key(raw: str | None) -> str:
|
||||
if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None:
|
||||
raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}")
|
||||
return raw
|
||||
|
||||
|
||||
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
|
||||
values = [v for k, v in query if k == name]
|
||||
if len(values) > 1:
|
||||
|
||||
Reference in New Issue
Block a user