From 3f12698676ba295450bf86409914bd7d3a129567 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:32:13 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20HAProxy=20=D0=B8=20N=20API-=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=BB=D0=B8=D0=BA,=20X-Instance-Id,=20=D0=BE=D0=B1=D1=89?= =?UTF-8?q?=D0=B0=D1=8F=20=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82=D1=8C,=20drain=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B8=20SIGTERM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - compose: один масштабируемый сервис api (deploy.replicas=2, без ports и container_name), HAProxy 3.2 LTS по digest — единственная точка входа (127.0.0.1), разовый migrate до старта реплик, CPU/RAM-лимиты у всех сервисов, каждому сервису только его secrets. - HAProxy: server-template + Docker DNS (re-resolve 1 с), активная проверка GET /health/ready каждую 1 с (fall 2 / rise 2), без backup-серверов, retry только conn-failure, ошибки самого LB — в формате контракта с X-Request-Id. - API: X-Instance-Id = hostname контейнера на каждом ответе (и на 500/503); Idempotency-Key обязателен; ключ, сообщение и результат пишутся в ОДНОЙ транзакции (миграция 0002), конкурентный повтор ждёт уникальный индекс; другой body — 409, посторонний — всегда 404; attachment_ids=[] равен отсутствию поля; дубли ключей JSON и тело у POST /auth/sessions — 400. - SIGTERM: readiness 503 + DRAIN_SECONDS=3 обслуживания, затем graceful stop. - scripts: stop-one.sh (+ .cmd для Windows), restore-api.sh, restart.sh перезапускает и LB, lbctl.py — состояние бэкендов HAProxy -> контейнеры. Co-Authored-By: Claude Opus 5.5 --- .env.example | 18 +++- Dockerfile | 2 +- Makefile | 30 +++++-- compose.yaml | 72 +++++++++++++--- infra/haproxy/haproxy.cfg | 89 +++++++++++++++++++ migrations/0002_idempotency_keys.sql | 25 ++++++ scripts/lbctl.py | 123 +++++++++++++++++++++++++++ scripts/restart.sh | 12 +-- scripts/restore-api.sh | 8 ++ scripts/stop-one.cmd | 7 ++ scripts/stop-one.sh | 35 ++++++++ src/messenger/__main__.py | 42 ++++++--- src/messenger/app.py | 77 ++++++++++++++--- src/messenger/config.py | 31 ++++++- src/messenger/repo.py | 87 ++++++++++++++++--- src/messenger/validation.py | 32 ++++++- 16 files changed, 629 insertions(+), 61 deletions(-) create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0002_idempotency_keys.sql create mode 100644 scripts/lbctl.py create mode 100755 scripts/restore-api.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index c84d354..c18efac 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,10 @@ DATABASE_URL_FILE=/run/secrets/database_url # valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`. SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url +# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas). +API_REPLICAS=2 +# Idempotency-Key results are kept at least this long (contract: >= 24 h). +IDEMPOTENCY_TTL_SECONDS=86400 # Additional settings of this implementation (defaults shown; all optional): LOG_LEVEL=info DB_POOL_MIN=1 @@ -19,5 +23,17 @@ DB_TIMEOUT_SECONDS=3 CACHE_TIMEOUT_SECONDS=0.5 PASSWORD_HASH_CONCURRENCY=2 DOCS_ENABLED=true -# Host port of the API (compose publishes it on 127.0.0.1 only) +# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests. +DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=8 +# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica. +API_CPUS=1.0 +API_MEMORY=256m +PG_CPUS=2.0 +PG_MEMORY=1g +VALKEY_CPUS=0.5 +LB_CPUS=1.0 +PG_MAX_CONNECTIONS=100 +# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing) APP_PUBLISH_PORT=8080 +LB_STATS_PORT=8404 diff --git a/Dockerfile b/Dockerfile index 470f37a..12bb32c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Messenger API, lab 2 (PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). +# Messenger API, lab 3 (replicas behind HAProxy, PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). # # Python is interpreted, so there is no separate compile stage; the image # contains only the interpreter, locked dependencies and the source code. diff --git a/Makefile b/Makefile index 6233ac4..58704a5 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,14 @@ test-failover: $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) # --------------------------------------------------------------------------- -# Student solution (lab 2, Python). On Windows without make run the commands from -# REPORT.md directly (public tests there need PYTHONUTF8=1; restart hook: scripts/restart.cmd). -.PHONY: secrets up down logs ps migrate student-test unit-test explain +# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public +# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash). +.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series secrets: $(PYTHON) scripts/init_secrets.py -# Build, apply migrations (service migrate) and wait until every service is healthy. +# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy. up: secrets docker compose up -d --build --wait @@ -46,14 +46,32 @@ down: docker compose down logs: - docker compose logs --tail 50 api + docker compose logs --tail 50 api lb ps: - docker compose ps + docker compose ps -a migrate: docker compose run --rm migrate +# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id. +lb-status: + $(PYTHON) scripts/lbctl.py status + $(PYTHON) scripts/lbctl.py replicas + +# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP. +restore: + sh scripts/restore-api.sh + $(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2} + +# Replace every API replica one by one with a freshly built image (no downtime). +rolling-update: + sh scripts/rolling-update.sh + +# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/). +failover-series: + $(PYTHON) bench/failover_series.py --output evidence/failover + # Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv). student-test: $(PYTHON) -m pytest student-tests -v diff --git a/compose.yaml b/compose.yaml index 1f76ebe..09abcfd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,9 +1,12 @@ -# Lab 2 stand: API + PostgreSQL + Valkey. Start from a clean clone: +# Lab 3 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) -> PostgreSQL + Valkey. +# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md): # python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) -# cp .env.example .env # optional overrides -# docker compose up -d --build --wait -# Images are pinned by version AND digest. Passwords are passed only as secret files. -name: mtusi-lab2 +# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...) +# docker compose --env-file .env up -d --build --wait +# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb. +# Images are pinned by version AND digest. Passwords are passed only as secret files, and +# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only). +name: mtusi-lab3 x-api-env: &api-env APP_ENV: ${APP_ENV:-lab} @@ -17,6 +20,10 @@ x-api-env: &api-env DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3} CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + # Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests. + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} DOCS_ENABLED: ${DOCS_ENABLED:-true} services: @@ -26,6 +33,8 @@ services: POSTGRES_USER: messenger POSTGRES_DB: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + # pg_stat_statements: per-statement counters for the track 5 experiments (bench/track1). + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] secrets: [postgres_password] volumes: - pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path @@ -37,6 +46,9 @@ services: start_period: 10s stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery restart: unless-stopped + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"} valkey: image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 @@ -61,10 +73,15 @@ services: retries: 30 start_period: 5s restart: unless-stopped + deploy: + resources: + limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} + # One-shot schema migration: runs ONCE per `up`, before any API replica starts + # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: build: . - image: mtusi-messenger:lab2 + image: mtusi-messenger:lab3 command: ["python", "-m", "messenger.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -75,14 +92,24 @@ services: healthcheck: disable: true restart: "no" + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} api: build: . - image: mtusi-messenger:lab2 + image: mtusi-messenger:lab3 environment: *api-env secrets: [database_url, cache_url] - ports: - - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # only on localhost of the host + # No `ports` and no container_name: replicas are reachable only through `lb`, + # and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname. + expose: ["8080"] + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck"] + interval: 2s + timeout: 2s + retries: 3 + start_period: 5s depends_on: postgres: condition: service_healthy @@ -90,8 +117,33 @@ services: condition: service_healthy migrate: condition: service_completed_successfully - stop_grace_period: 10s + stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) restart: unless-stopped + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-256m}"} + + lb: + image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # the ONLY public entry point + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas) + healthcheck: + # Config is valid and the process answers; whether any API is ready is the LB's own job. + test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"] + interval: 5s + timeout: 3s + retries: 3 + depends_on: + api: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} volumes: pgdata: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..09aff87 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,89 @@ +# HAProxy 3.2 LTS — the single public entry point of the lab 3 stand (compose service `lb`). +# +# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the +# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and +# re-resolves every second, so scale up/down, a restarted container with a new IP and a +# full stack restart are picked up without restarting the LB. A stopped container drops +# out of DNS and its slot goes to MAINT. +# +# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey reachable, not draining) +# every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide anything on +# their own, and there are NO backup/fallback servers without checks: a replica that answers +# /health/live but not /health/ready never gets business traffic. +# Worst case detection of a failed replica: +# inter 1 s + fall 2 x (check timeout 1 s) ~= 3 s (frozen process); +# refused connection / 503 ready: inter 1 s + fastinter 0.5 s ~= 1.5 s. +# Requests that could not even be SENT (connection refused/reset before the request) +# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST, +# because the API has not seen them. Nothing that may have reached the API is retried by +# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key. +# No sticky sessions: sessions live in Valkey, every replica serves every user. + +global + log stdout format raw local0 info + maxconn 4096 + # Runtime API for `show servers state` / `show stat` (scripts and tests, no network port). + stats socket /var/lib/haproxy/admin.sock mode 600 level admin + hard-stop-after 15s + +resolvers docker + nameserver dns1 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + +defaults + mode http + log global + option httplog + option dontlognull + option http-keep-alive + timeout connect 1s + timeout client 30s + timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1s + retries 2 + retry-on conn-failure + option redispatch 1 + # Errors generated by the LB itself, in the contract error format with the same request id. + # They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the + # request is unknown (a replica died mid-request / did not answer in time): retry with the + # same Idempotency-Key. 503 = no ready replica at all. + http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + +frontend public + bind :8080 + # One request id end to end: keep a valid client X-Request-Id, otherwise create one; the + # API echoes it, and errors generated by the LB itself carry the same value. + http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ } + http-request set-var(txn.request_id) req.hdr(x-request-id) + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\"" + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 + server-template api 8 api:8080 + +# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now. +frontend stats + bind :8404 + stats enable + stats uri / + stats refresh 2s + stats show-legends diff --git a/migrations/0002_idempotency_keys.sql b/migrations/0002_idempotency_keys.sql new file mode 100644 index 0000000..108dafd --- /dev/null +++ b/migrations/0002_idempotency_keys.sql @@ -0,0 +1,25 @@ +-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages. +-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe +-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции"). +-- +-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key. +-- The method (POST) and the path template are fixed for this table, so the scope +-- is (user_id, chat_id, key). The key row, the message and the result are written in +-- ONE transaction (repo.post_message): there is no "pending" state that a crash could +-- leave behind, and the stored result is the message row itself. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body + -- Inserted before the message in the same transaction, hence checked at COMMIT. + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_message_key UNIQUE (message_id) +); +-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h). +CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at); diff --git a/scripts/lbctl.py b/scripts/lbctl.py new file mode 100644 index 0000000..755d4c6 --- /dev/null +++ b/scripts/lbctl.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""What the load balancer thinks right now, mapped to real containers. + + python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id + python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy + python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state + +Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`, +so no admin port is published. Container hostname = X-Instance-Id of that replica. +Prints no secrets. Used by student-tests and bench scripts as a library as well. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping. +OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"} +ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it) + + +def run(*args: str, timeout: float = 60) -> str: + result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8", + timeout=timeout) + if result.returncode != 0: + raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}") + return result.stdout + + +def replicas(all_states: bool = True) -> list[dict]: + """API containers of this compose project (running and stopped).""" + ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split() + if not ids: + return [] + data = json.loads(run("docker", "inspect", *ids)) + result = [] + for item in data: + networks = item["NetworkSettings"]["Networks"] or {} + result.append({ + "id": item["Id"][:12], + "name": item["Name"].lstrip("/"), + "hostname": item["Config"]["Hostname"], + "state": item["State"]["Status"], + "health": (item["State"].get("Health") or {}).get("Status", "-"), + "exit_code": item["State"]["ExitCode"], + "started_at": item["State"]["StartedAt"], + "pid": item["State"]["Pid"], + "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), + }) + return sorted(result, key=lambda r: r["name"]) + + +def lb_servers() -> list[dict]: + raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c", + 'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock') + by_ip = {ip: r for r in replicas() for ip in r["ips"]} + servers = [] + for line in raw.splitlines(): + parts = line.split() + if len(parts) < 7 or not parts[0].isdigit(): + continue + name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6]) + if admin & ADMIN_RMAINT or addr == "-": + state = "MAINT" + else: + state = OP_STATE.get(op, str(op)) + container = by_ip.get(addr) + servers.append({"slot": name, "addr": addr, "state": state, + "container": container["name"] if container else None, + "instance_id": container["hostname"] if container else None}) + return servers + + +def up_instances() -> set[str]: + return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"} + + +def wait_up(count: int, timeout: float = 60) -> float: + """Seconds until exactly `count` servers are UP (raises TimeoutError otherwise).""" + started = time.monotonic() + while True: + try: + if len(up_instances()) == count: + return time.monotonic() - started + except RuntimeError: + pass # lb restarting + if time.monotonic() - started > timeout: + raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s") + time.sleep(0.25) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + sub.add_parser("replicas") + wait = sub.add_parser("wait-up") + wait.add_argument("count", type=int) + wait.add_argument("--timeout", type=float, default=60) + args = parser.parse_args() + if args.command == "status": + for s in lb_servers(): + if s["state"] != "MAINT" or s["container"]: + print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}") + print(f"(other slots: MAINT, no container)") + elif args.command == "replicas": + for r in replicas(): + print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} " + f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}") + else: + took = wait_up(args.count, args.timeout) + print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh index 973048a..b641607 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,15 +1,17 @@ #!/bin/sh # Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. -# Restarts the API and ALL state stores (PostgreSQL, Valkey) of compose.yaml. +# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL and Valkey. # Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. # Output is suppressed by the test; run it by hand to see the steps. set -eu cd "$(dirname "$0")/.." -# 1. API first: no request is served while the stores restart underneath it. -docker compose stop api +# 1. Entry point and API first: no request is served while the stores restart underneath. +docker compose stop lb api # 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again. docker compose restart postgres valkey -# 3. API again; it waits for healthy stores via depends_on and reconnects its pool. +# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves +# the new replica addresses through Docker DNS by itself - no stale backend list. docker compose start api -echo "restart.sh: api, postgres and valkey restarted with volumes preserved" +docker compose start lb +echo "restart.sh: lb, api (all replicas), postgres and valkey restarted with volumes preserved" diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 0000000..78a0684 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker +# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s); +# `python scripts/lbctl.py wait-up 2` waits for that explicitly. +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-recreate --no-deps --wait api +docker compose ps api --format '{{.Name}} {{.Status}}' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..1292249 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows +rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash; +rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment. +rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..85bc37a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`. +# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname, +# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are +# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy); +# restore it explicitly afterwards: scripts/restore-api.sh. +# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s) +# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process +# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed). +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac +mode="${STOP_MODE:-stop}" + +target="" +# Only RUNNING containers of THIS compose project's service `api` are candidates. +for id in $(docker compose ps -q api); do + if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then + target="$id" + fi +done +if [ -z "$target" ]; then + echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2 + exit 3 +fi +name="$(docker inspect -f '{{.Name}}' "$target")" +case "$mode" in + stop) docker stop --time 15 "$target" >/dev/null ;; + kill) docker stop --time 0 "$target" >/dev/null ;; + *) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;; +esac +echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py index 331a1fd..5701b66 100644 --- a/src/messenger/__main__.py +++ b/src/messenger/__main__.py @@ -1,16 +1,20 @@ """Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker). -Shutdown sequence on SIGTERM/SIGINT: -1. readiness flips to 503 (`app.state.draining = True`); -2. uvicorn closes the listening socket, so new connections are refused; -3. in-flight requests get up to SHUTDOWN_TIMEOUT_SECONDS (default 8 s < Docker's 10 s) to finish; +Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy): +1. readiness flips to 503 (`app.state.draining = True`), requests are still served; +2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has + marked this replica DOWN and sends nothing new here; +3. uvicorn closes the listening socket; in-flight requests get up to + SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish; 4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0. - Data stays in PostgreSQL and Valkey volumes. + DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL. +A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey. """ from __future__ import annotations import logging import sys +import threading from types import FrameType import uvicorn @@ -23,12 +27,30 @@ log = logging.getLogger("messenger") class GracefulServer(uvicorn.Server): - """uvicorn.Server that marks the app as draining before the normal graceful stop.""" + """uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop.""" + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + self.drain_timer: threading.Timer | None = None def handle_exit(self, sig: int, frame: FrameType | None) -> None: - if not self.should_exit: - log.info("shutdown_requested", extra={"fields": {"signal": sig, "timeout_s": self.config.timeout_graceful_shutdown}}) self.config.app.state.draining = True # config.app is our Starlette instance + if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit: + log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds, + "timeout_s": self.config.timeout_graceful_shutdown}}) + # uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe. + self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame)) + self.drain_timer.daemon = True + self.drain_timer.start() + return + if self.drain_timer is not None: + self.drain_timer.cancel() # second signal: stop draining now + self._stop(sig, frame) + + def _stop(self, sig: int, frame: FrameType | None) -> None: + if not self.should_exit: + log.info("shutdown_requested", extra={"fields": {"signal": sig}}) super().handle_exit(sig, frame) @@ -51,10 +73,10 @@ def main() -> int: lifespan="on", timeout_graceful_shutdown=settings.shutdown_timeout, ) - server = GracefulServer(config) + server = GracefulServer(config, settings.drain_seconds) log.info("starting", extra={"fields": { "version": __version__, "env": settings.env, "host": settings.host, "port": settings.port, - "python": sys.version.split()[0], "storage": "postgresql", + "python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id, }}) server.run() log.info("stopped") diff --git a/src/messenger/app.py b/src/messenger/app.py index 9d0bf79..e2bb683 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio import contextlib +import hashlib +import json import logging import re import time @@ -19,7 +21,7 @@ from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Sen from . import authhdr, cursor, docs, passwords, repo, validation from .config import Settings from .db import Database -from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, PayloadTooLarge, Unauthorized, Unavailable +from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable from .sessions import SessionStore, build_store from .timefmt import rfc3339 @@ -56,10 +58,12 @@ async def http_exception_handler(request: Request, exc: Exception) -> JSONRespon # Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions) # -------------------------------------------------------------------------- class RequestContextMiddleware: - """Assigns X-Request-Id to every response, writes one access-log line, turns crashes into 500.""" + """Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503 + included), writes one access-log line, turns crashes into 500.""" - def __init__(self, app: ASGIApp): + def __init__(self, app: ASGIApp, instance_id: str): self.app = app + self.instance_id = instance_id.encode("ascii") async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] != "http": @@ -77,8 +81,10 @@ class RequestContextMiddleware: if message["type"] == "http.response.start": started = True status = message["status"] - headers = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"] + headers = [(k, v) for k, v in message.get("headers", []) + if k.lower() not in (b"x-request-id", b"x-instance-id")] headers.append((b"x-request-id", request_id.encode("latin-1"))) + headers.append((b"x-instance-id", self.instance_id)) message = {**message, "headers": headers} await send(message) @@ -90,20 +96,21 @@ class RequestContextMiddleware: if not started: status = 503 response = error_response(request_id, 503, "unavailable", "Сервер завершает работу") - await response(scope, receive, send) + await response(scope, receive, send_with_id) raise except Exception: log.exception("unhandled_error", extra={"fields": {"request_id": request_id}}) if not started: status = 500 response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера") - await response(scope, receive, send) + await response(scope, receive, send_with_id) finally: access_log.info("request", extra={"fields": { "request_id": request_id, "method": scope["method"], "path": scope["path"], # ids only; query (cursor) and bodies are never logged "status": status, + "instance_id": self.instance_id.decode("ascii"), "duration_ms": round((time.perf_counter() - begin) * 1000, 3), }}) @@ -166,6 +173,27 @@ def page_params(request: Request) -> tuple[int, str | None]: return limit, raw_cursor +async def reject_body(request: Request) -> None: + """POST /auth/sessions takes credentials only from the Basic header; any body is 400.""" + if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0": + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + async for chunk in request.stream(): + if chunk: + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + + +def idempotency_key(request: Request) -> str: + values = request.headers.getlist("idempotency-key") + return validation.idempotency_key(values[0] if len(values) == 1 else None) + + +def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes: + """SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter.""" + canonical = json.dumps({"attachment_ids": attachment_ids, "text": text}, + ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + def ok(body: Any, status: int = 200) -> JSONResponse: return JSONResponse(body, status_code=status) @@ -205,6 +233,7 @@ async def register_user(request: Request) -> JSONResponse: async def create_session(request: Request) -> JSONResponse: """Login: only `Authorization: Basic`; wrong user and wrong password give the same 401.""" username, password = authhdr.parse_basic(request.headers.get("authorization")) + await reject_body(request) found = await repo.get_credentials(db_of(request), username) user, stored_hash = found if found else (None, None) if not await passwords.verify_password(stored_hash, password) or user is None: @@ -262,13 +291,23 @@ async def create_message(request: Request) -> JSONResponse: user_id = await authenticate(request) chat_id = path_uuid(request, "chat_id") db = db_of(request) - await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before the body is read + await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at + key = idempotency_key(request) data = await read_json(request) - validation.check_fields(data, {"text"}) + validation.check_fields(data, {"text"}, frozenset({"attachment_ids"})) text = validation.text_field(data, "text", 4000) + attachment_ids = validation.attachment_ids_field(data) + if attachment_ids: + # Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404). + raise NotFound("Вложение не найдено") # sender_id is derived from the session, never taken from the body. - message = await repo.add_message(db, user_id, chat_id, text) - return ok(message.to_json(), 201) + settings = request.app.state.settings + result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text, + settings.idempotency_ttl_seconds) + if result.replayed: + log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id, + "message_id": result.message.id}}) + return ok(result.message.to_json(), 201) async def list_messages(request: Request) -> JSONResponse: @@ -288,16 +327,30 @@ def create_app(settings: Settings | None = None) -> Starlette: db = Database(settings) sessions = build_store(settings, db) + async def purge_loop() -> None: + """Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS).""" + while True: + await asyncio.sleep(600) + try: + deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds) + if deleted: + log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}}) + except ApiError: + pass # database unavailable right now; next round + @contextlib.asynccontextmanager async def lifespan(app: Starlette): passwords.configure(settings.hash_concurrency) await db.open() log.info("dependencies_configured", extra={"fields": { "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, - "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds}}) + "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, + "instance_id": settings.instance_id, "drain_s": settings.drain_seconds}}) + purge = asyncio.create_task(purge_loop()) try: yield finally: + purge.cancel() await sessions.close() await db.close() @@ -324,7 +377,7 @@ def create_app(settings: Settings | None = None) -> Starlette: exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler}, lifespan=lifespan, ) - app.add_middleware(RequestContextMiddleware) + app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id) app.state.settings = settings app.state.db = db app.state.sessions = sessions diff --git a/src/messenger/config.py b/src/messenger/config.py index 4cecbcd..807020c 100644 --- a/src/messenger/config.py +++ b/src/messenger/config.py @@ -8,6 +8,8 @@ Invalid values stop the process at startup with a clear message. from __future__ import annotations import os +import re +import socket from dataclasses import dataclass from pathlib import Path @@ -16,6 +18,22 @@ class ConfigError(ValueError): pass +INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") + + +def _instance_id() -> str: + """X-Instance-Id of this process: INSTANCE_ID or the container hostname. + + Docker sets the hostname of a container to its short container ID, so the value + is unique per replica, stable for the life of the process and maps 1:1 to a real + container (`docker inspect`), which scripts/stop-one.sh relies on. + """ + value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + if INSTANCE_ID_RE.fullmatch(value) is None: + raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}") + return value + + def _int(name: str, default: int, low: int, high: int) -> int: raw = os.environ.get(name) if raw is None or raw.strip() == "": @@ -85,6 +103,9 @@ class Settings: db_timeout_seconds: float = 3.0 cache_timeout_seconds: float = 0.5 hash_concurrency: int = 2 + instance_id: str = "local" + drain_seconds: float = 0.0 + idempotency_ttl_seconds: int = 86400 @property def db_statement_timeout_ms(self) -> int: @@ -110,8 +131,8 @@ class Settings: port=_int("APP_PORT", 8080, 1, 65535), env=os.environ.get("APP_ENV", "lab").strip() or "lab", log_level=log_level, - # Docker sends SIGKILL 10 s after SIGTERM by default; finish earlier. - shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 9.0), + # DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period. + shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0), max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024), docs_enabled=_bool("DOCS_ENABLED", True), database_url=_secret("DATABASE_URL", required=True), @@ -124,4 +145,10 @@ class Settings: db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0), cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0), hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16), + instance_id=_instance_id(), + # After SIGTERM: readiness is 503 but requests are still served for this long, + # so the load balancer stops routing here before the listener closes. + drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0), + # Contract: a replay within 24 h returns the original result. + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600), ) diff --git a/src/messenger/repo.py b/src/messenger/repo.py index 4ae4d95..71bb0c6 100644 --- a/src/messenger/repo.py +++ b/src/messenger/repo.py @@ -15,7 +15,7 @@ from psycopg import errors as pg_errors from .cursor import SortKey from .db import Database -from .errors import Conflict, InvalidRequest, NotFound +from .errors import Conflict, InvalidRequest, NotFound, Unavailable from .timefmt import from_us, rfc3339, to_us @@ -198,28 +198,89 @@ async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: i # ------------------------------------------------------------------ messages -async def add_message(db: Database, user_id: str, chat_id: str, text: str) -> Message: +@dataclass(slots=True, frozen=True) +class PostResult: + message: Message + replayed: bool # True: the key was already used with this body, nothing new was written + + +async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str, + text: str) -> dt.datetime: """Insert with a per-chat strictly increasing timestamp. `UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one chat are serialised: a message with a smaller created_at can never commit after a message with a bigger one, and a reader paging by cursor cannot skip it. """ + cur = await conn.execute( + "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " + "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", + (chat_id,)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (message_id, chat_id, user_id, _b(text), created_at)) + return created_at + + +# Claim the key. A row of a concurrent transaction that is not committed yet makes this +# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict +# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is +# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired). +CLAIM_KEY = ( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) " + "VALUES (%s, %s, %s, %s, %s) " + "ON CONFLICT (user_id, chat_id, key) DO UPDATE SET " + "request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() " + "WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) " + "RETURNING 1") +STORED_RESULT = ( + "SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k " + "JOIN messages m ON m.id = k.message_id " + "WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s") + + +async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str, + ttl_seconds: int) -> PostResult: + """Idempotent send: the key, the message and the stored result commit in ONE transaction. + + * first request -> key row + message + chat clock, all or nothing (no "pending" key is + ever left behind: a crash or 5xx rolls everything back and the client may retry); + * same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed; + * same key, other body -> 409; the membership check comes first, so outsiders always get 404. + """ message_id = uuid.uuid4() - async def work(conn: psycopg.AsyncConnection) -> dt.datetime: + async def work(conn: psycopg.AsyncConnection) -> PostResult: await _check_member(conn, user_id, chat_id) + cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds)) + if await cur.fetchone() is not None: + created_at = await _insert_message(conn, message_id, user_id, chat_id, text) + return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False) + cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key)) + row = await cur.fetchone() + if row is None: + # The key row vanished between the two statements (expired-key cleanup): retryable. + raise Unavailable("Повторите запрос с тем же Idempotency-Key") + if bytes(row[0]) != request_hash: + raise Conflict("Idempotency-Key уже использован с другим сообщением") + return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True) + return await db.run(work) + + +async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int: + """Delete expired keys in bounded batches; one replica at a time (advisory xact lock).""" + async def work(conn: psycopg.AsyncConnection) -> int: + cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,)) + if not (await cur.fetchone())[0]: + return 0 cur = await conn.execute( - "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " - "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", - (chat_id,)) - (created_at,) = await cur.fetchone() - await conn.execute( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", - (message_id, chat_id, user_id, _b(text), created_at)) - return created_at - created_at = await db.run(work) - return Message(str(message_id), chat_id, user_id, text, created_at) + "DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY(" + "SELECT ctid FROM idempotency_keys " + "WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))", + (ttl_seconds, batch)) + return cur.rowcount + return await db.run(work) # Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR diff --git a/src/messenger/validation.py b/src/messenger/validation.py index 8767095..b338c9e 100644 --- a/src/messenger/validation.py +++ b/src/messenger/validation.py @@ -16,6 +16,8 @@ USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") # Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case. UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") LIMIT_RE = re.compile(r"[0-9]{1,3}") +# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") SURROGATES_RE = re.compile("[\ud800-\udfff]") DEFAULT_LIMIT = 50 @@ -27,11 +29,19 @@ def _reject_constant(name: str) -> Any: raise ValueError(f"invalid JSON constant {name}") +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + # {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins". + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError("duplicate key in JSON object") + return result + + def parse_json_object(body: bytes) -> dict[str, Any]: """Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400).""" try: text = body.decode("utf-8") - value = json.loads(text, parse_constant=_reject_constant) + value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys) except (UnicodeDecodeError, ValueError): raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None if not isinstance(value, dict): @@ -102,6 +112,26 @@ def member_ids_field(data: dict[str, Any]) -> list[str]: return ids +def attachment_ids_field(data: dict[str, Any]) -> list[str]: + """Optional `attachment_ids` (lab 4+). Absent and [] are the same message (contracts/README.md); + lab 3 has no attachments yet, so a non-empty list can only reference unknown objects.""" + value = data.get("attachment_ids", []) + if not isinstance(value, list): + raise InvalidRequest("attachment_ids должен быть массивом") + if len(value) > 10: + raise InvalidRequest("attachment_ids содержит больше 10 элементов") + ids = [parse_uuid(item, "UUID вложения") for item in value] + if len(set(ids)) != len(ids): + raise InvalidRequest("attachment_ids содержит дубликаты") + return ids + + +def idempotency_key(raw: str | None) -> str: + if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None: + raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}") + return raw + + def single_query_param(query: list[tuple[str, str]], name: str) -> str | None: values = [v for k, v in query if k == name] if len(values) > 1: