From ae53d39ccc0fe11f97548135d79571820b67ee65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D1=8F=20=D0=9C=D0=BE=D1=80=D0=BE=D0=B7?= <145702378+MiniLynx13@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:35:44 +0300 Subject: [PATCH] feat:level4 --- Dockerfile | 2 +- Makefile | 8 ++- cmd/messenger/main.go | 5 +- compose.yaml | 96 ++++++++++++++++++++++++----- infra/nginx/nginx.conf | 28 +++++++++ migrations/0004_idempotency.sql | 14 +++++ scripts/restart.sh | 14 +++++ scripts/restore-one.sh | 9 +++ scripts/stop-one.sh | 15 +++++ src/api/handlers.go | 46 +++++++++++++- src/api/middleware.go | 11 ++++ src/api/router.go | 6 +- src/config/config.go | 17 +++++ src/domain/idempotency.go | 75 ++++++++++++++++++++++ src/domain/postgres.go | 93 ++++++++++++++++++++++++++++ student_tests/common.py | 67 ++++++++++++++++++++ student_tests/test_failover_both.py | 87 ++++++++++++++++++++++++++ student_tests/test_idempotency.py | 88 ++++++++++++++++++++++++++ student_tests/test_limits.py | 41 ++++++++++++ student_tests/test_rolling.py | 79 ++++++++++++++++++++++++ 20 files changed, 775 insertions(+), 26 deletions(-) create mode 100644 infra/nginx/nginx.conf create mode 100644 migrations/0004_idempotency.sql create mode 100644 scripts/restart.sh create mode 100644 scripts/restore-one.sh create mode 100644 scripts/stop-one.sh create mode 100644 src/domain/idempotency.go create mode 100644 student_tests/common.py create mode 100644 student_tests/test_failover_both.py create mode 100644 student_tests/test_idempotency.py create mode 100644 student_tests/test_limits.py create mode 100644 student_tests/test_rolling.py diff --git a/Dockerfile b/Dockerfile index a4387c7..040241e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,6 @@ USER app EXPOSE 8080 HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \ - CMD wget -qO- http://127.0.0.1:8080/health/live >/dev/null || exit 1 + CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 ENTRYPOINT ["/app/messenger"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d17..cfe72aa 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover my_tests help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,9 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +my_tests: + $(PYTHON) student_tests/test_limits.py + $(PYTHON) student_tests/test_idempotency.py + $(PYTHON) student_tests/test_failover_both.py + $(PYTHON) student_tests/test_rolling.py \ No newline at end of file diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go index c9399d3..37448ff 100644 --- a/cmd/messenger/main.go +++ b/cmd/messenger/main.go @@ -50,7 +50,7 @@ func main() { sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL) handler := &api.Handler{Store: store, Sessions: sessions} - router := api.NewRouter(handler) + router := api.NewRouter(handler, cfg.InstanceID) srv := &http.Server{ Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), @@ -66,6 +66,7 @@ func main() { go func() { log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL) + log.Printf("instance_id=%s", cfg.InstanceID) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("server error: %v", err) } @@ -73,7 +74,7 @@ func main() { <-stop log.Println("shutting down...") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _ = srv.Shutdown(ctx) log.Println("stopped") diff --git a/compose.yaml b/compose.yaml index 2460b9a..d05fe19 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,34 @@ +x-api-base: &api-base + build: + context: . + dockerfile: Dockerfile + env_file: + - .env + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: "lab" + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_STORE: "postgres" + SESSION_TTL_SECONDS: "3600" + secrets: + - database_url + - cache_url + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + networks: [backend] + deploy: + resources: + limits: + cpus: "0.75" + memory: 256M + restart_policy: + condition: on-failure + services: postgres: image: postgres:16.3-alpine @@ -15,6 +46,11 @@ services: timeout: 3s retries: 20 networks: [backend] + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M redis: image: redis:7.2.5-alpine @@ -27,30 +63,35 @@ services: timeout: 3s retries: 20 networks: [backend] + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M - api: - build: - context: . - dockerfile: Dockerfile - env_file: - - .env + api1: + <<: *api-base environment: APP_HOST: "0.0.0.0" APP_PORT: "8080" APP_ENV: "lab" DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url - secrets: - - database_url - - cache_url - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - ports: - - "8080:8080" - networks: [backend] + SESSION_STORE: "postgres" + SESSION_TTL_SECONDS: "3600" + INSTANCE_ID: "api-1" + + api2: + <<: *api-base + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: "lab" + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_STORE: "postgres" + SESSION_TTL_SECONDS: "3600" + INSTANCE_ID: "api-2" migrate: build: @@ -67,6 +108,27 @@ services: networks: [backend] restart: "no" + lb: + image: nginx:1.27.0-alpine + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "8080:8080" + depends_on: + - api1 + - api2 + networks: [backend] + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/live"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + volumes: pgdata: redisdata: diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 0000000..4fb385b --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,28 @@ +worker_processes auto; +events { worker_connections 1024; } + +http { + upstream api_backend { + server api1:8080 max_fails=2 fail_timeout=5s; + server api2:8080 max_fails=2 fail_timeout=5s; + keepalive 32; + } + + server { + listen 8080; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + proxy_connect_timeout 2s; + proxy_send_timeout 15s; + proxy_read_timeout 15s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + } + } +} \ No newline at end of file diff --git a/migrations/0004_idempotency.sql b/migrations/0004_idempotency.sql new file mode 100644 index 0000000..da62120 --- /dev/null +++ b/migrations/0004_idempotency.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys (created_at); \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..22f8b16 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +# Перезапуск API + всех state stores, volumes сохраняются. +docker compose restart postgres redis api1 api2 lb +# Ждём готовности. +for i in $(seq 1 60); do + if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then + echo "ready after ${i}s" + exit 0 + fi + sleep 1 +done +echo "stack did not become ready in 60s" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/restore-one.sh b/scripts/restore-one.sh new file mode 100644 index 0000000..deab0d1 --- /dev/null +++ b/scripts/restore-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +case "$TARGET_INSTANCE_ID" in + api-1) SERVICE=api1 ;; + api-2) SERVICE=api2 ;; + *) exit 2 ;; +esac +docker compose start "$SERVICE" \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..5a102f6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +# TARGET_INSTANCE_ID приходит от теста: alias обслужившей реплики. +# В compose alias = INSTANCE_ID = "api-1" / "api-2". +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +case "$TARGET_INSTANCE_ID" in + api-1) SERVICE=api1 ;; + api-2) SERVICE=api2 ;; + *) echo "unknown instance: $TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac + +# Останавливаем ТОЛЬКО эту реплику, оставляем остановленной. +docker compose stop "$SERVICE" +echo "stopped $SERVICE ($TARGET_INSTANCE_ID)" \ No newline at end of file diff --git a/src/api/handlers.go b/src/api/handlers.go index b874d80..d244c6a 100644 --- a/src/api/handlers.go +++ b/src/api/handlers.go @@ -1,8 +1,10 @@ package api import ( + "bytes" "encoding/json" "errors" + "io" "net/http" "regexp" "strconv" @@ -12,6 +14,7 @@ import ( ) var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) type Handler struct { Store *domain.Store @@ -224,12 +227,31 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { me, _ := userFrom(r) chatID := r.PathValue("chat_id") + + // Idempotency-Key обязателен. + key := r.Header.Get("Idempotency-Key") + if key == "" { + writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required") + return + } + if !idempotencyKeyRe.MatchString(key) { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key") + return + } + if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil { writeError(w, r, http.StatusNotFound, "not_found", "chat not found") return } + + // Читаем тело один раз и хешируем нормализованный JSON. + rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024)) + if err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body") + return + } var req createMessageReq - dec := json.NewDecoder(r.Body) + dec := json.NewDecoder(bytes.NewReader(rawBody)) dec.DisallowUnknownFields() if err := dec.Decode(&req); err != nil { writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") @@ -243,13 +265,23 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message") return } - m, err := h.Store.AddMessage(r.Context(), chatID, me.ID, req.Text) + bodyHash, err := domain.HashBody(rawBody) + if err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") + return + } + + msg, rec, replay, err := h.Store.AddMessageIdempotent( + r.Context(), chatID, me.ID, req.Text, key, bodyHash) switch { case err == nil: // ok case errors.Is(err, domain.ErrNotFound): writeError(w, r, http.StatusNotFound, "not_found", "chat not found") return + case errors.Is(err, domain.ErrConflict): + writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body") + return case errors.Is(err, domain.ErrTimeout): writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") return @@ -257,7 +289,15 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") return } - writeJSON(w, http.StatusCreated, m) + + if replay { + // Возвращаем сохранённый 201 и то же Message. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(rec.StatusCode) + _, _ = w.Write(rec.Response) + return + } + writeJSON(w, http.StatusCreated, msg) })(w, r) } diff --git a/src/api/middleware.go b/src/api/middleware.go index 932bab4..7cce92e 100644 --- a/src/api/middleware.go +++ b/src/api/middleware.go @@ -33,3 +33,14 @@ func withRequestID(next http.Handler) http.Handler { next.ServeHTTP(w, r.WithContext(ctx)) }) } + +// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health). +// ID генерируется один раз при старте процесса — стабилен в пределах жизни. +func withInstanceID(instanceID string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", instanceID) + next.ServeHTTP(w, r) + }) + } +} diff --git a/src/api/router.go b/src/api/router.go index 26a85f3..a9e9f90 100644 --- a/src/api/router.go +++ b/src/api/router.go @@ -2,7 +2,7 @@ package api import "net/http" -func NewRouter(h *Handler) http.Handler { +func NewRouter(h *Handler, instanceID string) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /health/live", h.Live) mux.HandleFunc("GET /health/ready", h.Ready) @@ -15,5 +15,7 @@ func NewRouter(h *Handler) http.Handler { mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat) mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage) mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages) - return withRequestID(mux) + + // Порядок: instanceID снаружи, requestID внутри. + return withInstanceID(instanceID)(withRequestID(mux)) } diff --git a/src/config/config.go b/src/config/config.go index 1b98813..430a2ec 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -1,6 +1,8 @@ package config import ( + "crypto/rand" + "encoding/hex" "os" "strconv" "time" @@ -15,6 +17,7 @@ type Config struct { SessionTTL time.Duration SessionStore string // "postgres" | "redis" RedisRequired bool + InstanceID string } func getenv(key, def string) string { @@ -59,6 +62,11 @@ func Load() Config { cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE")) } + instanceID := os.Getenv("INSTANCE_ID") + if instanceID == "" { + instanceID = "api-" + randomShortID() + } + store := getenv("SESSION_STORE", "postgres") return Config{ Host: getenv("APP_HOST", "0.0.0.0"), @@ -69,5 +77,14 @@ func Load() Config { SessionTTL: time.Duration(ttl) * time.Second, SessionStore: store, RedisRequired: store == "redis", + InstanceID: instanceID, } } + +func randomShortID() string { + b := make([]byte, 6) + if _, err := rand.Read(b); err != nil { + return "unknown" + } + return hex.EncodeToString(b) +} diff --git a/src/domain/idempotency.go b/src/domain/idempotency.go new file mode 100644 index 0000000..f340f77 --- /dev/null +++ b/src/domain/idempotency.go @@ -0,0 +1,75 @@ +package domain + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// IdempotencyRecord — сохранённый результат POST. +type IdempotencyRecord struct { + StatusCode int + Response json.RawMessage +} + +// LookupIdempotency возвращает запись, если ключ уже использован. +// Второй аргумент — совпадает ли хеш тела с сохранённым. +func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) { + var ( + storedHash string + rec IdempotencyRecord + ) + err := s.pool.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`, + userID, method, path, key, + ).Scan(&storedHash, &rec.StatusCode, &rec.Response) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, classifyDBError(err) + } + if storedHash != bodyHash { + return nil, true, ErrConflict + } + return &rec, true, nil +} + +// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный +// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency. +func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error { + _, err := s.pool.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1,$2,$3,$4,$5,$6,$7)`, + userID, method, path, key, bodyHash, status, resp) + return classifyDBError(err) +} + +// HashBody — стабильный хеш JSON независимо от порядка ключей. +func HashBody(raw []byte) (string, error) { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return "", err + } + normalized, err := json.Marshal(v) // map keys сортируются + if err != nil { + return "", err + } + sum := sha256.Sum256(normalized) + return hex.EncodeToString(sum[:]), nil +} + +// CleanupIdempotency — можно звать из фонового тикера (опционально). +func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error { + _, err := s.pool.Exec(ctx, + `DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`, + olderThan.String()) + return err +} diff --git a/src/domain/postgres.go b/src/domain/postgres.go index 0dfb868..0ad7b41 100644 --- a/src/domain/postgres.go +++ b/src/domain/postgres.go @@ -10,6 +10,7 @@ import ( "crypto/rand" "crypto/subtle" "encoding/base64" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -443,3 +444,95 @@ func classifyDBError(err error) error { } return err } + +// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение +// и сохраняет результат идемпотентности в одной транзакции. +// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true. +func (s *Store) AddMessageIdempotent( + ctx context.Context, + chatID, senderID, text, key, bodyHash string, +) (msg *Message, rec *IdempotencyRecord, replay bool, err error) { + path := "/api/v1/chats/" + chatID + "/messages" + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + return nil, nil, false, classifyDBError(err) + } + defer tx.Rollback(ctx) + + // 1) Членство. + var ok bool + if err := tx.QueryRow(ctx, + `SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`, + chatID, senderID, + ).Scan(&ok); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil, false, ErrNotFound + } + return nil, nil, false, classifyDBError(err) + } + + // 2) Уже был такой ключ? + var storedHash string + var storedStatus int + var storedResp json.RawMessage + err = tx.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`, + senderID, path, key, + ).Scan(&storedHash, &storedStatus, &storedResp) + switch { + case err == nil: + if storedHash != bodyHash { + return nil, nil, false, ErrConflict + } + return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil + case !errors.Is(err, pgx.ErrNoRows): + return nil, nil, false, classifyDBError(err) + } + + // 3) Вставка сообщения. + var m Message + err = tx.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1,$2,$3,$4) + RETURNING id, chat_id, sender_id, text, created_at`, + uuid.NewString(), chatID, senderID, text, + ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + if err != nil { + // Гонка на одном ключе в разных репликах: unique violation в idempotency_keys + // маловероятен здесь, но serialization failure возможен. + return nil, nil, false, classifyDBError(err) + } + + // 4) Сохранение ответа. + respJSON, _ := json.Marshal(m) + _, err = tx.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1,'POST',$2,$3,$4,201,$5)`, + senderID, path, key, bodyHash, respJSON) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + // Кто-то параллельно успел вставить: перечитываем в этой же tx. + if err2 := tx.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`, + senderID, path, key, + ).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil { + if storedHash != bodyHash { + return nil, nil, false, ErrConflict + } + return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil + } + } + return nil, nil, false, classifyDBError(err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, nil, false, classifyDBError(err) + } + return &m, nil, false, nil +} diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 0000000..ea854d2 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,67 @@ +"""Общие утилиты для студенческих тестов.""" +import base64 +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) + +from client import Client, Failure, check # noqa: E402 + +BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080') +COMPOSE_CMD = os.environ.get('COMPOSE_CMD', 'docker compose') +COMPOSE_PROJECT = os.environ.get('COMPOSE_PROJECT', '') + + +def compose(*args, timeout=120, check_rc=True): + """Вызов docker compose. Возвращает CompletedProcess.""" + cmd = COMPOSE_CMD.split() + if COMPOSE_PROJECT: + cmd += ['-p', COMPOSE_PROJECT] + cmd += list(args) + proc = subprocess.run( + cmd, cwd=str(ROOT), capture_output=True, text=True, timeout=timeout) + if check_rc and proc.returncode != 0: + raise Failure( + 'compose %s failed rc=%d\nstdout=%s\nstderr=%s' + % (' '.join(args), proc.returncode, proc.stdout[-2000:], proc.stderr[-2000:])) + return proc + + +def wait_ready(deadline_seconds=90): + c = Client(BASE_URL, timeout=2) + start = time.monotonic() + while time.monotonic() - start < deadline_seconds: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + time.sleep(0.5) + raise Failure('LB did not become ready in %ds' % deadline_seconds) + + +def observe_instances(c, actor, n=40): + """Собираем X-Instance-Id через /users/me.""" + seen = {} + for _ in range(n): + r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + iid = r['headers'].get('X-Instance-Id') + check(iid, 'X-Instance-Id missing') + seen[iid] = seen.get(iid, 0) + 1 + return seen + + +def run_file(path, *args, timeout=180): + proc = subprocess.run( + [sys.executable, str(path), *args], + cwd=str(ROOT), capture_output=True, text=True, timeout=timeout) + return proc + + +def print_pass(name, extra=''): + print('PASS: ' + name + (' ' + extra if extra else '')) \ No newline at end of file diff --git a/student_tests/test_failover_both.py b/student_tests/test_failover_both.py new file mode 100644 index 0000000..95bb66f --- /dev/null +++ b/student_tests/test_failover_both.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Останавливает КАЖДУЮ реплику по очереди и проверяет восстановление.""" +import json +import subprocess +import sys +import time +from pathlib import Path + +from common import ( + BASE_URL, Client, Failure, check, compose, observe_instances, print_pass, + wait_ready, +) + +ROOT = Path(__file__).resolve().parents[1] + + +def stop_one(instance_alias): + env = {'TARGET_INSTANCE_ID': instance_alias, 'PATH': __import__('os').environ['PATH']} + proc = subprocess.run( + [str(ROOT / 'scripts/stop-one.sh')], + cwd=str(ROOT), env={**__import__('os').environ, **env}, + capture_output=True, text=True, timeout=30) + check(proc.returncode == 0, + 'stop-one.sh для %s упал rc=%d\nstderr=%s' + % (instance_alias, proc.returncode, proc.stderr[-1000:])) + + +def restore_all(): + """Возвращает все реплики в running.""" + for svc in ('api1', 'api2'): + compose('start', svc, check_rc=False) + wait_ready(90) + + +def main(): + c = Client(BASE_URL) + alice, bob, eve, chat = c.bootstrap() + seen = observe_instances(c, alice, n=40) + check(len(seen) >= 2, 'видна одна реплика: %r' % seen) + print(' instances:', seen) + + for alias in sorted(seen): + print('--- failover test for %s' % alias) + target_alias = alias + stop_one(target_alias) + time.sleep(1) + + # Ни одна последующая успешная запись не должна обслуживаться target. + first_ok = None + after = [] + deadline = time.monotonic() + 15 + key = 'fo-' + __import__('uuid').uuid4().hex + while time.monotonic() < deadline: + try: + r = c.message(alice, chat, 'after-stop-' + key, key=key) + if first_ok is None: + first_ok = time.monotonic() + after.append(r['headers']['X-Instance-Id']) + key = 'fo-' + __import__('uuid').uuid4().hex + except Failure: + pass + time.sleep(0.2) + check(first_ok is not None, 'запись не восстановилась за 15с') + check(len(after) >= 5, 'слишком мало успешных записей после отказа: %d' % len(after)) + check(target_alias not in after[-5:], + 'остановленная реплика %s всё ещё отвечает' % target_alias) + print(' after stop %s: %d writes, последние instances=%r' + % (target_alias, len(after), after[-3:])) + + # Восстановить и подождать, пока LB снова начнёт её использовать. + restore_all() + time.sleep(2) + seen2 = observe_instances(c, alice, n=60) + print(' после restore видно:', seen2) + + # В конце — обе реплики снова живы. + seen3 = observe_instances(c, alice, n=60) + check(len(seen3) >= 2, 'после restore видна одна реплика: %r' % seen3) + print_pass('failover both replicas') + + +if __name__ == '__main__': + try: + main() + except Failure as e: + print('FAIL:', e) + raise SystemExit(1) \ No newline at end of file diff --git a/student_tests/test_idempotency.py b/student_tests/test_idempotency.py new file mode 100644 index 0000000..9e9fb92 --- /dev/null +++ b/student_tests/test_idempotency.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Проверки идемпотентности и X-Instance-Id (лаб3).""" +import json +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import ( + BASE_URL, Client, Failure, check, observe_instances, print_pass, +) + + +def main(): + c = Client(BASE_URL) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + + # 1) Через LB видно ≥2 реплики. + seen = observe_instances(c, alice, n=40) + check(len(seen) >= 2, 'менее двух X-Instance-Id через LB: %r' % seen) + print(' instances:', seen) + + # 2) Повтор с тем же ключом и тем же телом → тот же Message. + key = 'idem-' + uuid.uuid4().hex + body = {'text': 'idem-verify-' + uuid.uuid4().hex} + r1 = c.call('POST', path, 201, actor=alice, body=body, + headers={'Idempotency-Key': key}, schema='Message') + r2 = c.call('POST', path, 201, actor=alice, body=body, + headers={'Idempotency-Key': key}, schema='Message') + check(r1['json']['id'] == r2['json']['id'], 'replay дал другой id') + check(r1['json'] == r2['json'], 'replay дал другое тело') + + # 3) Тот же ключ, другое тело → 409. + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, + headers={'Idempotency-Key': key}) + + # 4) Нет ключа → 400. + c.call('POST', path, 400, actor=alice, body={'text': 'no key'}) + + # 5) Невалидный ключ (пробел) → 400. + c.call('POST', path, 400, actor=alice, body={'text': 'bad key'}, + headers={'Idempotency-Key': 'has space'}) + + # 6) Разные пользователи с одинаковым ключом — независимы. + key_shared = 'shared-' + uuid.uuid4().hex + ra = c.message(alice, chat, 'alice-' + uuid.uuid4().hex, key=key_shared) + rb = c.message(bob, chat, 'bob-' + uuid.uuid4().hex, key=key_shared) + check(ra['json']['id'] != rb['json']['id'], + 'одинаковый ключ у разных пользователей склеил сообщения') + + # 7) Конкурентный повтор: 20 параллельных POST с одним ключом. + key_c = 'conc-' + uuid.uuid4().hex + text_c = 'concurrent-' + uuid.uuid4().hex + + def send(_): + local = Client(BASE_URL, timeout=15) + # Разрешаем 201 или 503 (transient serialization failure). + r = local.call('POST', path, (201, 503), actor=alice, + body={'text': text_c}, + headers={'Idempotency-Key': key_c}) + payload = r['json'] + if payload is None and r['status'] == 201: + try: + payload = json.loads(r['bytes']) + except ValueError: + payload = None + return {'status': r['status'], 'json': payload} + + with ThreadPoolExecutor(max_workers=20) as pool: + results = list(pool.map(send, range(20))) + + ok = [r for r in results if r['status'] == 201 and r['json']] + check(len(ok) >= 15, 'слишком много не-201 при конкурентном повторе: %d/20' + % len(ok)) + ids = {r['json']['id'] for r in ok} + check(len(ids) == 1, + 'конкурентный повтор создал %d разных сообщений: %r' + % (len(ids), ids)) + print(' concurrent: %d×201, один id' % len(ok)) + + print_pass('idempotency') + + +if __name__ == '__main__': + try: + main() + except Failure as e: + print('FAIL:', e) + raise SystemExit(1) \ No newline at end of file diff --git a/student_tests/test_limits.py b/student_tests/test_limits.py new file mode 100644 index 0000000..5066132 --- /dev/null +++ b/student_tests/test_limits.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Проверяет, что контейнеры имеют CPU/RAM лимиты и api не публикует порт.""" +import json + +from common import Failure, check, compose, print_pass + + +def main(): + raw = compose('config', '--format', 'json').stdout + cfg = json.loads(raw) + services = cfg['services'] + + for svc in ('api1', 'api2'): + check(svc in services, 'сервис %s отсутствует' % svc) + ports = services[svc].get('ports', []) + check(not ports, '%s публикует порты наружу: %r — должно быть только через LB' + % (svc, ports)) + deploy = services[svc].get('deploy', {}) + res = deploy.get('resources', {}).get('limits', {}) + check('cpus' in res and 'memory' in res, + '%s без deploy.resources.limits cpus/memory: %r' % (svc, res)) + + check('lb' in services, 'нет сервиса lb') + lb_ports = services['lb'].get('ports', []) + check(any('8080' in str(p) for p in lb_ports), + 'lb не публикует 8080: %r' % lb_ports) + + check('container_name' not in services.get('api1', {}), + 'api1 задаёт container_name — ломает масштабирование') + check('container_name' not in services.get('api2', {}), + 'api2 задаёт container_name — ломает масштабирование') + + print_pass('limits & topology') + + +if __name__ == '__main__': + try: + main() + except Failure as e: + print('FAIL:', e) + raise SystemExit(1) \ No newline at end of file diff --git a/student_tests/test_rolling.py b/student_tests/test_rolling.py new file mode 100644 index 0000000..64157b4 --- /dev/null +++ b/student_tests/test_rolling.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Rolling update одной реплики под нагрузкой (уровень 4).""" +import json +import threading +import time +import uuid + +from common import ( + BASE_URL, Client, Failure, check, compose, print_pass, +) + + +def main(): + c = Client(BASE_URL) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + + stop = threading.Event() + stats = {'ok': 0, 'err': 0, 'instances': {}} + + def load(): + local = Client(BASE_URL, timeout=5) + while not stop.is_set(): + try: + key = uuid.uuid4().hex + r = local.call('POST', path, 201, actor=alice, + body={'text': 'roll-' + key}, + headers={'Idempotency-Key': key}, + schema='Message') + stats['ok'] += 1 + iid = r['headers'].get('X-Instance-Id') + stats['instances'][iid] = stats['instances'].get(iid, 0) + 1 + except Failure: + stats['err'] += 1 + time.sleep(0.05) + + t = threading.Thread(target=load, daemon=True) + t.start() + time.sleep(10) + base_ok = stats['ok'] + base_err = stats['err'] + print(' base: ok=%d err=%d' % (base_ok, base_err)) + + # Rolling update api1. + compose('up', '-d', '--no-deps', '--force-recreate', 'api1', timeout=180) + time.sleep(15) + + stop.set() + t.join(timeout=10) + + delta_ok = stats['ok'] - base_ok + delta_err = stats['err'] - base_err + print(' after rolling: ok=+%d err=+%d instances=%r' + % (delta_ok, delta_err, stats['instances'])) + + check(delta_ok > 0, 'rolling update остановил запись полностью') + # Ошибки допустимы кратковременно, но не должны доминировать. + check(delta_err <= max(5, delta_ok // 3), + 'слишком много ошибок при rolling update: +%d ok / +%d err' + % (delta_ok, delta_err)) + + # После rolling обе реплики должны снова обслуживать. + seen_final = {} + for _ in range(40): + r = c.call('GET', '/api/v1/users/me', actor=alice, schema='User') + iid = r['headers']['X-Instance-Id'] + seen_final[iid] = seen_final.get(iid, 0) + 1 + check(len(seen_final) >= 2, + 'после rolling видна одна реплика: %r' % seen_final) + print(' final instances:', seen_final) + print_pass('rolling update under load') + + +if __name__ == '__main__': + try: + main() + except Failure as e: + print('FAIL:', e) + raise SystemExit(1) \ No newline at end of file