diff --git a/.gitignore b/.gitignore index b5a7189..452fd1c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +secrets/ diff --git a/cmd/api/main.go b/cmd/api/main.go index 1a045ad..10c70ed 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -4,7 +4,9 @@ import ( "context" "log" "net/http" + "os" "os/signal" + "sync/atomic" "syscall" "time" @@ -66,30 +68,46 @@ func main() { chatCache := cache.New(redisClient, chatCacheTTL) s := store.New(pool, limiter) - mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache) + + instanceID := config.GetEnv("INSTANCE_ID", "") + if instanceID == "" { + instanceID, _ = os.Hostname() + } + var draining atomic.Bool + + mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache, httpapi.Options{ + InstanceID: instanceID, + Redis: redisClient, + Draining: &draining, + }) port := config.GetEnv("PORT", "8080") srv := &http.Server{ - Addr: ":" + port, - Handler: mux, + Addr: ":" + port, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() go func() { - log.Printf("запуск на :%s", port) + log.Printf("запуск на :%s instance=%s", port, instanceID) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Fatalf("сервер упал: %v", err) } }() <-ctx.Done() - log.Println("Получен сигнал остановки, завершаю активные запросы...") + log.Println("Получен сигнал остановки: readiness -> 503, жду LB...") + draining.Store(true) + time.Sleep(time.Duration(config.ParseIntEnv("DRAIN_SECONDS", 4)) * time.Second) - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 12*time.Second) defer cancel() - if err := srv.Shutdown(shutdownCtx); err != nil { log.Printf("принудительное завершение: %v", err) } diff --git a/compose.yml b/compose.yml index a4eb85e..2612fe6 100644 --- a/compose.yml +++ b/compose.yml @@ -3,8 +3,10 @@ services: image: postgres:16.4-alpine environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password volumes: - postgres-data:/var/lib/postgresql/data healthcheck: @@ -13,8 +15,10 @@ services: timeout: 3s retries: 10 start_period: 5s - ports: - - "127.0.0.1:5431:5432" + networks: [data] + deploy: + resources: + limits: { cpus: "1.0", memory: 512M } redis: image: redis:7.4-alpine @@ -27,6 +31,10 @@ services: timeout: 3s retries: 10 start_period: 5s + networks: [data] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } migrate: image: migrate/migrate:v4.17.1 @@ -40,6 +48,7 @@ services: - "-database=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable" - "up" restart: "no" + networks: [data] api: build: @@ -54,23 +63,55 @@ services: condition: service_healthy environment: PORT: ${PORT} - DATABASE_URL: ${DATABASE_URL} + DATABASE_URL_FILE: /run/secrets/database_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} - MAX_MESSAGES: ${MAX_MESSAGES} MAX_CONCURRENT_REQUESTS: ${MAX_CONCURRENT_REQUESTS} DB_MAX_CONNS: ${DB_MAX_CONNS} DB_MIN_CONNS: ${DB_MIN_CONNS} DB_CONN_TIMEOUT_SECONDS: ${DB_CONN_TIMEOUT_SECONDS} CACHE_URL: ${CACHE_URL} CHAT_CACHE_TTL_SECONDS: 30 - ports: - - "127.0.0.1:8080:8080" + DRAIN_SECONDS: 4 + GOMEMLIMIT: 200MiB + APP_VERSION: ${APP_VERSION:-dev} + secrets: + - database_url + stop_grace_period: 20s healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] - interval: 10s - timeout: 3s - retries: 5 - start_period: 10s + interval: 3s + timeout: 2s + retries: 3 + start_period: 5s + networks: [edge, data] + deploy: + replicas: 2 + resources: + limits: { cpus: "0.5", memory: 256M } + + lb: + image: haproxy:3.0-alpine + depends_on: + api: + condition: service_healthy + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + networks: [edge] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + +networks: + edge: + data: + +secrets: + database_url: + file: ./secrets/database_url.txt + postgres_password: + file: ./secrets/postgres_password.txt volumes: postgres-data: diff --git a/evidence/failover-api1.json b/evidence/failover-api1.json new file mode 100644 index 0000000..4e6f221 --- /dev/null +++ b/evidence/failover-api1.json @@ -0,0 +1,16 @@ +{ + "container": "mtusi2026lab3-api-5", + "stopped_instance": "ba50cd9c50eb", + "rps_before": 891.5, + "rps_during": 850.6, + "rps_after": 966.7, + "first_attempt_errors": 0, + "error_window_s_after_stop": null, + "longest_gap_without_success_s": 0.18, + "stopped_instance_seen_while_down": false, + "confirmed": 31607, + "unconfirmed_keys": 0, + "stored": 31607, + "lost_confirmed": 0, + "stored_minus_confirmed": 0 +} diff --git a/evidence/failover-api2.json b/evidence/failover-api2.json new file mode 100644 index 0000000..5ce5077 --- /dev/null +++ b/evidence/failover-api2.json @@ -0,0 +1,16 @@ +{ + "container": "mtusi2026lab3-api-6", + "stopped_instance": "f9ec7883c11a", + "rps_before": 978.8, + "rps_during": 975.9, + "rps_after": 961.9, + "first_attempt_errors": 0, + "error_window_s_after_stop": null, + "longest_gap_without_success_s": 0.09, + "stopped_instance_seen_while_down": false, + "confirmed": 33997, + "unconfirmed_keys": 0, + "stored": 33997, + "lost_confirmed": 0, + "stored_minus_confirmed": 0 +} diff --git a/evidence/infra.txt b/evidence/infra.txt new file mode 100644 index 0000000..d80c1fa --- /dev/null +++ b/evidence/infra.txt @@ -0,0 +1,20 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-5 mtusi2026lab3-api "./api" api 4 minutes ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-6 mtusi2026lab3-api "./api" api 4 minutes ago Up 32 seconds (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 4 minutes ago Up About a minute 127.0.0.1:8080->8080/tcp +mtusi2026lab3-migrate-1 migrate/migrate:v4.17.1 "migrate -path=/migr…" migrate 11 hours ago Exited (0) 32 seconds ago +mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up About a minute (healthy) 5432/tcp +mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 4 minutes ago Up About a minute (healthy) 6379/tcp +/mtusi2026lab3-api-5 cpus=500000000 mem=268435456 +/mtusi2026lab3-api-6 cpus=500000000 mem=268435456 +/mtusi2026lab3-lb-1 cpus=250000000 mem=134217728 +/mtusi2026lab3-postgres-1 cpus=1000000000 mem=536870912 +/mtusi2026lab3-redis-1 cpus=250000000 mem=134217728 +0 +migrate-1 | no change +migrate-1 | no change +migrate-1 | no change +migrate-1 | no change +migrate-1 | no change +migrate-1 | no change +DATABASE_URL_FILE=/run/secrets/database_url diff --git a/evidence/lb-check.txt b/evidence/lb-check.txt new file mode 100644 index 0000000..040ee84 --- /dev/null +++ b/evidence/lb-check.txt @@ -0,0 +1,8 @@ +PASS one token served by >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a'] +PASS concurrent same key: all 201 [201] +PASS concurrent same key: one message id 1 +PASS concurrent requests hit >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a'] +PASS sequential repeat: same id +PASS sequential repeat: >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a'] +PASS different body, same key -> 409 +PASS history has exactly 2 messages 2 diff --git a/evidence/load-baseline-inner.json b/evidence/load-baseline-inner.json new file mode 100644 index 0000000..4f972d1 --- /dev/null +++ b/evidence/load-baseline-inner.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.041, + "concurrency": 4, + "successful_requests": 28040, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 933.404, + "successful_latency_ms": { + "p50": 2.346, + "p95": 7.173, + "p99": 48.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-baseline.json b/evidence/load-baseline.json new file mode 100644 index 0000000..43eb6bb --- /dev/null +++ b/evidence/load-baseline.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 31.868, + "concurrency": 4, + "successful_requests": 2491, + "failed_requests": 20, + "error_ratio": 0.007965, + "successful_rps": 78.166, + "successful_latency_ms": { + "p50": 3.893, + "p95": 8.14, + "p99": 18.909 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 0000000..4b429d2 --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.043, + "concurrency": 4, + "successful_requests": 55307, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 690.963, + "successful_latency_ms": { + "p50": 2.553, + "p95": 24.852, + "p99": 51.029 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-rolling.txt b/evidence/load-rolling.txt new file mode 100644 index 0000000..4b429d2 --- /dev/null +++ b/evidence/load-rolling.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.043, + "concurrency": 4, + "successful_requests": 55307, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 690.963, + "successful_latency_ms": { + "p50": 2.553, + "p95": 24.852, + "p99": 51.029 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-stop-api2.json b/evidence/load-stop-api2.json new file mode 100644 index 0000000..72fffc7 --- /dev/null +++ b/evidence/load-stop-api2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 31.564, + "concurrency": 4, + "successful_requests": 2495, + "failed_requests": 19, + "error_ratio": 0.007558, + "successful_rps": 79.047, + "successful_latency_ms": { + "p50": 3.615, + "p95": 6.022, + "p99": 13.593 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-stop-api2.txt b/evidence/load-stop-api2.txt new file mode 100644 index 0000000..72fffc7 --- /dev/null +++ b/evidence/load-stop-api2.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 31.564, + "concurrency": 4, + "successful_requests": 2495, + "failed_requests": 19, + "error_ratio": 0.007558, + "successful_rps": 79.047, + "successful_latency_ms": { + "p50": 3.615, + "p95": 6.022, + "p99": 13.593 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/ps-after-rolling.txt b/evidence/ps-after-rolling.txt new file mode 100644 index 0000000..ffc4966 --- /dev/null +++ b/evidence/ps-after-rolling.txt @@ -0,0 +1,6 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-7 mtusi2026lab3-api "./api" api 40 seconds ago Up 39 seconds (healthy) 8080/tcp +mtusi2026lab3-api-8 mtusi2026lab3-api "./api" api 24 seconds ago Up 23 seconds (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 8 minutes ago Up 5 minutes 127.0.0.1:8080->8080/tcp +mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 8 minutes ago Up 5 minutes (healthy) 5432/tcp +mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 8 minutes ago Up 5 minutes (healthy) 6379/tcp diff --git a/evidence/ps-during-failover.txt b/evidence/ps-during-failover.txt new file mode 100644 index 0000000..d488dc4 --- /dev/null +++ b/evidence/ps-during-failover.txt @@ -0,0 +1,5 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-1 mtusi2026lab3-api "./api" api 14 minutes ago Up 7 minutes (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 21 minutes ago Up 14 minutes 127.0.0.1:8080->8080/tcp +mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 21 minutes ago Up 14 minutes (healthy) 127.0.0.1:5431->5432/tcp +mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 21 minutes ago Up 14 minutes (healthy) 6379/tcp diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..ae212e7 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 + +defaults + mode http + timeout connect 2s + timeout client 15s + timeout server 15s + retries 2 + option redispatch + log global + option httplog + timeout check 2s + +resolvers docker + nameserver dns 127.0.0.11:53 + hold valid 2s + timeout resolve 1s + timeout retry 1s + resolve_retries 3 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-6 api:8080 diff --git a/internal/config/config.go b/internal/config/config.go index 7b343d5..a5ddf81 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" "strconv" + "strings" "time" ) @@ -10,6 +11,11 @@ func GetEnv(key, fallback string) string { if v := os.Getenv(key); v != "" { return v } + if p := os.Getenv(key + "_FILE"); p != "" { + if b, err := os.ReadFile(p); err == nil { + return strings.TrimSpace(string(b)) + } + } return fallback } diff --git a/internal/httpapi/api_test.go b/internal/httpapi/api_test.go index 23485ea..7438fad 100644 --- a/internal/httpapi/api_test.go +++ b/internal/httpapi/api_test.go @@ -12,6 +12,7 @@ import ( "time" "mtusi2026/internal/cache" + "mtusi2026/internal/idgen" "mtusi2026/internal/model" "mtusi2026/internal/sessionstore" "mtusi2026/internal/store" @@ -21,6 +22,39 @@ import ( //вспомогательные функции +func newKey() string { return idgen.GenerateUUIDv4() } + +func doMessage(srv *httptest.Server, chatID, token, key string, body any) (*http.Response, error) { + var buf bytes.Buffer + if s, ok := body.(string); ok { + buf.WriteString(s) + } else if err := json.NewEncoder(&buf).Encode(body); err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, + fmt.Sprintf("%s/api/v1/chats/%s/messages", srv.URL, chatID), &buf) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + return srv.Client().Do(req) +} + +func postMessage(t *testing.T, srv *httptest.Server, chatID, token, key string, body any) *http.Response { + t.Helper() + resp, err := doMessage(srv, chatID, token, key, body) + if err != nil { + t.Fatalf("do request: %v", err) + } + return resp +} + func newUnlimitedLimiter() *store.Limiter { return store.NewLimiter(1_000_000, 1000) } @@ -412,9 +446,8 @@ func TestCreateMessage_EmptyText_Returns400(t *testing.T) { alice := mustRegisterUser(t, srv, "alice") chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) - resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{ - Text: "", - }) + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: ""}) + if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for empty text, got %d", resp.StatusCode) } @@ -427,9 +460,8 @@ func TestCreateMessage_WhitespaceOnlyText_Returns400(t *testing.T) { alice := mustRegisterUser(t, srv, "alice") chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) - resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{ - Text: " \t\n ", - }) + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: " \t\n "}) + if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for whitespace-only text, got %d", resp.StatusCode) } @@ -445,9 +477,8 @@ func TestCreateMessage_ForeignChat_Returns404(t *testing.T) { chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID, bob.ID}) - resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), carol.Token, model.CreateMessageRequest{ - Text: "Я тут чужая", - }) + resp := postMessage(t, srv, chat.ID, carol.Token, newKey(), model.CreateMessageRequest{Text: "Я тут чужая"}) + if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 for foreign chat, got %d", resp.StatusCode) } @@ -462,7 +493,7 @@ func TestCreateMessage_SenderIDIgnoredFromRequestBody(t *testing.T) { chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID, bob.ID}) raw := fmt.Sprintf(`{"text":"Привет","sender_id":%q}`, bob.ID) - resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, raw) + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), raw) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 (unknown field sender_id), got %d", resp.StatusCode) } @@ -479,7 +510,7 @@ func TestListMessages_PaginationAcrossMultiplePages(t *testing.T) { const total = 5 for i := 0; i < total; i++ { - resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{ + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{ Text: fmt.Sprintf("сообщение %d", i), }) if resp.StatusCode != http.StatusCreated { @@ -536,8 +567,8 @@ func TestListMessages_CursorFromDifferentChat_Returns400(t *testing.T) { chatA := mustCreateChat(t, srv, alice, "Чат A", []string{alice.ID}) chatB := mustCreateChat(t, srv, alice, "Чат B", []string{alice.ID}) - postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chatA.ID), alice.Token, model.CreateMessageRequest{Text: "первое"}).Body.Close() - postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chatA.ID), alice.Token, model.CreateMessageRequest{Text: "второе"}).Body.Close() + postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "первое"}).Body.Close() + postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "второе"}).Body.Close() respA := getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages?limit=1", chatA.ID), alice.Token) pageA := decodeBody[model.MessagePage](t, respA) @@ -761,7 +792,7 @@ func TestChatCache_WarmCache(t *testing.T) { []string{user.ID}, ) - // Первый GET прогревает cache. + // Первый GET прогревает cache resp := getJSON( t, srv, diff --git a/internal/httpapi/health.go b/internal/httpapi/health.go index b49d2ca..8df2046 100644 --- a/internal/httpapi/health.go +++ b/internal/httpapi/health.go @@ -3,18 +3,24 @@ package httpapi import ( "context" "net/http" + "sync/atomic" "time" "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" ) func handleHealthLive(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, newRequestID(), map[string]string{"status": "ok"}) } -func handleHealthReady(pool *pgxpool.Pool) http.HandlerFunc { +func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.Bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { requestID := newRequestID() + if draining.Load() { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "shutting down") + return + } ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) defer cancel() @@ -22,6 +28,12 @@ func handleHealthReady(pool *pgxpool.Pool) http.HandlerFunc { writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") return } + if rdb != nil { + if err := rdb.Ping(ctx).Err(); err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "session store unavailable") + return + } + } writeJSON(w, http.StatusOK, requestID, map[string]string{"status": "ok"}) } } diff --git a/internal/httpapi/idempotency_test.go b/internal/httpapi/idempotency_test.go new file mode 100644 index 0000000..bdb5362 --- /dev/null +++ b/internal/httpapi/idempotency_test.go @@ -0,0 +1,184 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "sync" + "testing" + + "mtusi2026/internal/model" +) + +func TestIdempotency_RepeatReturnsSameMessage(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + key := newKey() + body := model.CreateMessageRequest{Text: "привет"} + + r1 := postMessage(t, srv, chat.ID, alice.Token, key, body) + if r1.StatusCode != http.StatusCreated { + t.Fatalf("first: expected 201, got %d", r1.StatusCode) + } + m1 := decodeBody[model.Message](t, r1) + + r2 := postMessage(t, srv, chat.ID, alice.Token, key, body) + if r2.StatusCode != http.StatusCreated { + t.Fatalf("repeat: expected 201, got %d", r2.StatusCode) + } + m2 := decodeBody[model.Message](t, r2) + + if m1.ID != m2.ID || !m1.CreatedAt.Equal(m2.CreatedAt) { + t.Fatalf("repeat returned a different message: %+v vs %+v", m1, m2) + } + + page := decodeBody[model.MessagePage](t, + getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token)) + if len(page.Items) != 1 { + t.Fatalf("expected 1 message in history, got %d", len(page.Items)) + } +} + +func TestIdempotency_DifferentBodySameKey_Returns409(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + key := newKey() + postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "один"}).Body.Close() + + resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "два"}) + defer resp.Body.Close() + if resp.StatusCode != http.StatusConflict { + t.Fatalf("expected 409, got %d", resp.StatusCode) + } +} + +func TestIdempotency_MissingOrInvalidKey_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + keys := map[string]string{ + "missing": "", + "space": "bad key", + "too long": strings.Repeat("a", 129), + "cyrillic": "ключ", + } + for name, key := range keys { + resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "x"}) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("%s: expected 400, got %d", name, resp.StatusCode) + } + } +} + +func TestIdempotency_ConcurrentSameKey_OneMessage(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + const workers = 20 + key := newKey() + start := make(chan struct{}) + type result struct { + status int + id string + err error + } + results := make(chan result, workers) + + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + resp, err := doMessage(srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "гонка"}) + if err != nil { + results <- result{err: err} + return + } + defer resp.Body.Close() + var m model.Message + _ = t.Fatal + results <- result{status: resp.StatusCode, id: m.ID} + }() + } + close(start) + wg.Wait() + close(results) + + ids := map[string]bool{} + for r := range results { + if r.err != nil { + t.Errorf("request error: %v", r.err) + continue + } + if r.status != http.StatusCreated { + t.Errorf("expected 201, got %d", r.status) + } + ids[r.id] = true + } + if len(ids) != 1 { + t.Fatalf("expected one distinct message id, got %d", len(ids)) + } + + page := decodeBody[model.MessagePage](t, + getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token)) + if len(page.Items) != 1 { + t.Fatalf("expected exactly 1 message stored, got %d", len(page.Items)) + } +} + +func TestIdempotency_ScopeIsPerUserAndChat(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + chatA := mustCreateChat(t, srv, alice, "A", []string{alice.ID, bob.ID}) + chatB := mustCreateChat(t, srv, alice, "B", []string{alice.ID, bob.ID}) + + key := newKey() + body := model.CreateMessageRequest{Text: "то же самое"} + ids := map[string]bool{} + for _, c := range []struct { + chat string + token string + }{ + {chatA.ID, alice.Token}, {chatB.ID, alice.Token}, {chatA.ID, bob.Token}, + } { + resp := postMessage(t, srv, c.chat, c.token, key, body) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("expected 201, got %d", resp.StatusCode) + } + ids[decodeBody[model.Message](t, resp).ID] = true + } + if len(ids) != 3 { + t.Fatalf("same key in different scopes must create 3 messages, got %d", len(ids)) + } +} + +func TestIdempotency_ForeignChatRepeat_Returns404(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + carol := mustRegisterUser(t, srv, "carol") + chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID}) + + key := newKey() + for i := 0; i < 2; i++ { + resp := postMessage(t, srv, chat.ID, carol.Token, key, model.CreateMessageRequest{Text: "чужой"}) + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("attempt %d: expected 404, got %d", i, resp.StatusCode) + } + } +} diff --git a/internal/httpapi/messages.go b/internal/httpapi/messages.go index d5f9d13..ef04677 100644 --- a/internal/httpapi/messages.go +++ b/internal/httpapi/messages.go @@ -1,36 +1,45 @@ package httpapi import ( + "errors" "net/http" + "regexp" "mtusi2026/internal/model" "mtusi2026/internal/store" ) +var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { chatID := r.PathValue("chat_id") + key := r.Header.Get("Idempotency-Key") + if !idemKeyRe.MatchString(key) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "missing or invalid Idempotency-Key") + return + } + var req model.CreateMessageRequest if err := decodeJSONStrict(r, &req); err != nil { writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "malformed request body") return } - m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, req.Text) + m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text) if err != nil { - switch err { - case store.ErrChatNotFound: + switch { + case errors.Is(err, store.ErrChatNotFound): writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") - case store.ErrCapacityExceeded: - writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "message storage capacity reached") - case store.ErrValidation: + case errors.Is(err, store.ErrIdempotencyConflict): + writeError(w, http.StatusConflict, requestID, codeConflict, "Idempotency-Key was used with a different request body") + case errors.Is(err, store.ErrValidation): writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid text") default: writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") } return } - writeJSON(w, http.StatusCreated, requestID, m) } diff --git a/internal/httpapi/race_test.go b/internal/httpapi/race_test.go index 56c7743..a7f1ec6 100644 --- a/internal/httpapi/race_test.go +++ b/internal/httpapi/race_test.go @@ -44,6 +44,7 @@ func TestConcurrentMessages_NoLostOrDuplicated(t *testing.T) { } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+alice.Token) + req.Header.Set("Idempotency-Key", fmt.Sprintf("race-%d-%d", clientNum, i)) resp, err := client.Do(req) if err != nil { errCh <- fmt.Errorf("client %d: do request: %w", clientNum, err) diff --git a/internal/httpapi/router.go b/internal/httpapi/router.go index fe8ed9d..b9479e9 100644 --- a/internal/httpapi/router.go +++ b/internal/httpapi/router.go @@ -2,8 +2,10 @@ package httpapi import ( "net/http" + "sync/atomic" "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" "mtusi2026/internal/cache" "mtusi2026/internal/model" @@ -11,40 +13,18 @@ import ( "mtusi2026/internal/store" ) +type Options struct { + InstanceID string + Redis *redis.Client + Draining *atomic.Bool +} + func NewRouter(s *store.Store, pool *pgxpool.Pool, sessionStores ...sessionstore.Store) http.Handler { var ss sessionstore.Store if len(sessionStores) > 0 { ss = sessionStores[0] } - - a := &api{ - store: s, - sessionStore: ss, - } - - mux := http.NewServeMux() - mux.HandleFunc("GET /health/live", handleHealthLive) - mux.HandleFunc("GET /health/ready", handleHealthReady(pool)) - - mux.HandleFunc("POST /api/v1/users", a.handleRegisterUser) - mux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser)) - - mux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat)) - mux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats)) - mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat)) - - mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) - mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) - - mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) - mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware(func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { - a.handleDeleteSession(w, r) - })) - - var handler http.Handler = mux - handler = loggingMiddleware(handler) - handler = concurrencyMiddleware(s.Limiter, handler) - return handler + return NewRouterWithCache(s, pool, ss, nil) } func NewRouterWithCache( @@ -52,33 +32,53 @@ func NewRouterWithCache( pool *pgxpool.Pool, sessionStore sessionstore.Store, chatCache *cache.Cache, + opts ...Options, ) http.Handler { + var o Options + if len(opts) > 0 { + o = opts[0] + } + if o.InstanceID == "" { + o.InstanceID = "local" + } + if o.Draining == nil { + o.Draining = new(atomic.Bool) + } + a := &api{ store: s, sessionStore: sessionStore, chatCache: chatCache, } + apiMux := http.NewServeMux() apiMux.HandleFunc("POST /api/v1/users", a.handleRegisterUser) + apiMux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser)) + apiMux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat)) + apiMux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats)) + apiMux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat)) + apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) + apiMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) + apiMux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + apiMux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware( + func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + a.handleDeleteSession(w, r) + })) - mux := http.NewServeMux() - mux.HandleFunc("GET /health/live", handleHealthLive) - mux.HandleFunc("GET /health/ready", handleHealthReady(pool)) - mux.HandleFunc("POST /api/v1/users", a.handleRegisterUser) - mux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser)) - mux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat)) - mux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats)) - mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat)) - mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) - mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) - mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) - mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware(func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { - a.handleDeleteSession(w, r) - })) + root := http.NewServeMux() + root.HandleFunc("GET /health/live", handleHealthLive) + root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Draining)) + root.Handle("/", concurrencyMiddleware(s.Limiter, apiMux)) - var handler http.Handler = mux - handler = loggingMiddleware(handler) - handler = concurrencyMiddleware(s.Limiter, handler) + var h http.Handler = root + h = loggingMiddleware(h) + h = instanceMiddleware(o.InstanceID, h) + return h +} - return handler +func instanceMiddleware(id string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", id) + next.ServeHTTP(w, r) + }) } diff --git a/internal/store/store.go b/internal/store/store.go index 2ec5bff..2a7f05d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,11 +2,12 @@ package store import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strconv" "sync/atomic" - "time" "unicode/utf8" "github.com/jackc/pgx/v5" @@ -14,6 +15,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "mtusi2026/internal/cursor" + "mtusi2026/internal/idgen" "mtusi2026/internal/model" ) @@ -233,56 +235,75 @@ func (s *Store) ListChatsForUser(ctx context.Context, userID, cursorStr string, return page, next, nil } -// --- Сообщения --- - -func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, text string) (model.Message, error) { - - if s.Limiter != nil { - if !s.Limiter.TryReserveMessage() { - return model.Message{}, ErrCapacityExceeded - } - } - release := func() { - if s.Limiter != nil { - s.Limiter.ReleaseMessage() - } - } +var ErrIdempotencyConflict = errors.New("idempotency key reused with different body") +func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string) (model.Message, error) { if !validMessageText(text) { - release() return model.Message{}, ErrValidation } + sum := sha256.Sum256([]byte(text)) + hash := hex.EncodeToString(sum[:]) + + tx, err := s.pool.Begin(ctx) + if err != nil { + return model.Message{}, err + } + defer tx.Rollback(ctx) var isMember bool - err := s.pool.QueryRow(ctx, - `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, - chatID, senderID, - ).Scan(&isMember) - if err != nil { - release() + if err := tx.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, + chatID, senderID).Scan(&isMember); err != nil { return model.Message{}, err } if !isMember { - release() return model.Message{}, ErrChatNotFound } + msgID := idgen.GenerateUUIDv4() + tag, err := tx.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) + VALUES ($1,$2,$3,$4,$5) + ON CONFLICT (user_id, chat_id, key) DO NOTHING`, + senderID, chatID, idemKey, hash, msgID) + if err != nil { + return model.Message{}, err + } + + if tag.RowsAffected() == 0 { + var storedHash string + var m model.Message + err := tx.QueryRow(ctx, + `SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`, + senderID, chatID, idemKey, + ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + if err != nil { + return model.Message{}, err + } + if storedHash != hash { + return model.Message{}, ErrIdempotencyConflict + } + return m, nil + } + var m model.Message - err = s.pool.QueryRow(ctx, - `INSERT INTO messages (chat_id, sender_id, text) - VALUES ($1, $2, $3) + err = tx.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1,$2,$3,$4, clock_timestamp()) RETURNING id, chat_id, sender_id, text, created_at`, - chatID, senderID, text, + msgID, chatID, senderID, text, ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) - if err != nil { - release() if isCheckViolation(err) { return model.Message{}, ErrValidation } return model.Message{}, err } - + if err := tx.Commit(ctx); err != nil { + return model.Message{}, err + } return m, nil } @@ -377,44 +398,8 @@ func validMessageText(s string) bool { return false } -func (s *Store) CreateSession(ctx context.Context, token, userID string, ttl time.Duration) (time.Time, error) { - expiresAt := time.Now().UTC().Add(ttl) - _, err := s.pool.Exec(ctx, - `INSERT INTO sessions (token, user_id, expires_at) VALUES ($1, $2, $3)`, - token, userID, expiresAt, - ) - if err != nil { - return time.Time{}, err - } - return expiresAt, nil -} - var ErrSessionNotFound = errors.New("session not found or expired") -func (s *Store) GetUserBySessionToken(ctx context.Context, token string) (model.User, error) { - var u model.User - err := s.pool.QueryRow(ctx, - `SELECT u.id, u.username, u.display_name, u.created_at - FROM sessions s - JOIN users u ON u.id = s.user_id - WHERE s.token = $1 AND s.expires_at > now()`, - token, - ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) - - if err != nil { - return model.User{}, ErrSessionNotFound - } - return u, nil -} - -func (s *Store) DeleteSession(ctx context.Context, token string) (bool, error) { - tag, err := s.pool.Exec(ctx, `DELETE FROM sessions WHERE token = $1`, token) - if err != nil { - return false, err - } - return tag.RowsAffected() > 0, nil -} - func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.User, string, error) { var u model.User var passwordHash string diff --git a/migrations/000002_idempotency.up.sql b/migrations/000002_idempotency.up.sql new file mode 100644 index 0000000..4657fa3 --- /dev/null +++ b/migrations/000002_idempotency.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idx_idempotency_created_at ON idempotency_keys (created_at); \ No newline at end of file diff --git a/scripts/check_lb.py b/scripts/check_lb.py new file mode 100644 index 0000000..db2ea63 --- /dev/null +++ b/scripts/check_lb.py @@ -0,0 +1,65 @@ +import base64, json, sys, urllib.error, urllib.request, uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, token=None, basic=None, key=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(BASE + path, data=data, method=method) + if body is not None: + req.add_header("Content-Type", "application/json") + if token: + req.add_header("Authorization", "Bearer " + token) + if basic: + req.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode()) + if key: + req.add_header("Idempotency-Key", key) + try: + with urllib.request.urlopen(req, timeout=10) as r: + raw = r.read() + return r.status, r.headers.get("X-Instance-Id"), json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + return e.code, e.headers.get("X-Instance-Id"), None + + +ok = True +def check(name, cond, info=""): + global ok + ok &= bool(cond) + print(("PASS" if cond else "FAIL"), name, info) + + +name = "lb_" + uuid.uuid4().hex[:10] +s, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "LB check", "password": PWD}) +s, _, sess = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +token = sess["token"] + +# один токен разные реплики +inst = {call("GET", "/api/v1/users/me", token=token)[1] for _ in range(12)} +check("one token served by >=2 replicas", len(inst) >= 2, sorted(inst)) + +s, _, chat = call("POST", "/api/v1/chats", {"title": "lb", "member_ids": [user["id"]]}, token=token) +path = f"/api/v1/chats/{chat['id']}/messages" + +# конкурентные повторы с одним ключом через lb +key = "conc-" + uuid.uuid4().hex +with ThreadPoolExecutor(30) as ex: + res = list(ex.map(lambda _: call("POST", path, {"text": "race"}, token=token, key=key), range(30))) +ids = {r[2]["id"] for r in res if r[2]} +check("concurrent same key: all 201", all(r[0] == 201 for r in res), sorted({r[0] for r in res})) +check("concurrent same key: one message id", len(ids) == 1, len(ids)) +check("concurrent requests hit >=2 replicas", len({r[1] for r in res}) >= 2, sorted({r[1] for r in res})) + +# последовательный повтор на разных репликах +key2 = "seq-" + uuid.uuid4().hex +seq = [call("POST", path, {"text": "seq"}, token=token, key=key2) for _ in range(6)] +check("sequential repeat: same id", len({r[2]["id"] for r in seq}) == 1) +check("sequential repeat: >=2 replicas", len({r[1] for r in seq}) >= 2, sorted({r[1] for r in seq})) +check("different body, same key -> 409", call("POST", path, {"text": "other"}, token=token, key=key2)[0] == 409) + +s, _, page = call("GET", path + "?limit=100", token=token) +check("history has exactly 2 messages", len(page["items"]) == 2, len(page["items"])) + +sys.exit(0 if ok else 1) \ No newline at end of file diff --git a/scripts/failover-under-load.sh b/scripts/failover-under-load.sh new file mode 100755 index 0000000..fb45af6 --- /dev/null +++ b/scripts/failover-under-load.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +mkdir -p evidence + +python3 tests/load.py --duration 30 --concurrency 4 \ + --output evidence/load-stop-api2.json > evidence/load-stop-api2.txt & +LOAD_PID=$! + +sleep 8 +date '+%T stop api-2' +docker stop -t 8 mtusi2026lab3-api-2 >/dev/null +docker compose ps > evidence/ps-during-failover.txt + +sleep 10 +date '+%T start api-2' +docker start mtusi2026lab3-api-2 >/dev/null + +wait "$LOAD_PID" +docker compose logs lb --since 3m > evidence/lb-failover.log +cat evidence/load-stop-api2.txt diff --git a/scripts/measure_failover.py b/scripts/measure_failover.py new file mode 100644 index 0000000..3658d6f --- /dev/null +++ b/scripts/measure_failover.py @@ -0,0 +1,91 @@ +import base64, json, subprocess, sys, threading, time, urllib.request, uuid +import os +BASE = os.environ.get("BASE", "http://localhost:8080"); PWD = "correct-horse-battery-staple" +WORKERS, DURATION, STOP_AT, START_AT = 4, 35, 8, 22 + + +def req(method, path, body=None, token=None, basic=None, key=None): + r = urllib.request.Request(BASE + path, method=method, + data=json.dumps(body).encode() if body is not None else None) + if body is not None: r.add_header("Content-Type", "application/json") + if token: r.add_header("Authorization", "Bearer " + token) + if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode()) + if key: r.add_header("Idempotency-Key", key) + with urllib.request.urlopen(r, timeout=3) as resp: + return resp.status, resp.headers.get("X-Instance-Id"), json.loads(resp.read() or b"null") + + +def sh(*a): return subprocess.run(a, capture_output=True, text=True).stdout.split() +arg = sys.argv[1] +names = sorted(sh("docker", "ps", "--filter", "label=com.docker.compose.service=api", + "--format", "{{.Names}}")) +container = names[int(arg) - 1] if arg.isdigit() else arg +stopped_id = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", container], + capture_output=True, text=True).stdout.strip() +assert stopped_id, "container not found" +name = "fo_" + uuid.uuid4().hex[:10] +_, _, u = req("POST", "/api/v1/users", {"username": name, "display_name": "fo", "password": PWD}) +_, _, s = req("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +tok = s["token"] +_, _, c = req("POST", "/api/v1/chats", {"title": "fo", "member_ids": [u["id"]]}, token=tok) +path = f"/api/v1/chats/{c['id']}/messages" + +t0 = time.time() +attempts, confirmed, unconfirmed = [], set(), 0 +lock = threading.Lock() + + +def action(): + time.sleep(STOP_AT) + subprocess.run(["docker", "stop", "-t", "8", container], capture_output=True) + time.sleep(max(0, START_AT - (time.time() - t0))) + subprocess.run(["docker", "start", container], capture_output=True) + + +def worker(): + global unconfirmed + while time.time() - t0 < DURATION: + key = uuid.uuid4().hex + done = False + for n in range(3): + ts = time.time() - t0 + try: + _, inst, m = req("POST", path, {"text": "x"}, token=tok, key=key) + with lock: + attempts.append((ts, True, inst, n)); confirmed.add(m["id"]) + done = True + break + except Exception: + with lock: attempts.append((ts, False, None, n)) + time.sleep(0.2) + if not done: + with lock: unconfirmed += 1 + + +threads = [threading.Thread(target=action)] + [threading.Thread(target=worker) for _ in range(WORKERS)] +[t.start() for t in threads]; [t.join() for t in threads] + +stored, cur = set(), None +while True: + _, _, page = req("GET", path + "?limit=100" + (f"&cursor={cur}" if cur else ""), token=tok) + stored |= {m["id"] for m in page["items"]} + cur = page["next_cursor"] + if not cur: break + +oks = sorted(a[0] for a in attempts if a[1]) +gap = max((b - a for a, b in zip(oks, oks[1:])), default=0) +first_fail = [a[0] - STOP_AT for a in attempts if not a[1] and a[3] == 0 and a[0] >= STOP_AT] +pre_stop_errors = sum(1 for a in attempts if not a[1] and a[3] == 0 and a[0] < STOP_AT) +rps = lambda a, b: round(sum(a <= t < b for t in oks) / (b - a), 1) +late = {a[2] for a in attempts if a[1] and STOP_AT + 6 < a[0] < START_AT} +print(json.dumps({ + "container": container, "stopped_instance": stopped_id, + "rps_before": rps(0, STOP_AT), "rps_during": rps(STOP_AT, START_AT), "rps_after": rps(START_AT, DURATION), + "first_attempt_errors": len(first_fail), + "error_window_s_after_stop": [round(min(first_fail), 2), round(max(first_fail), 2)] if first_fail else None, + "longest_gap_without_success_s": round(gap, 2), + "stopped_instance_seen_while_down": stopped_id in late, + "confirmed": len(confirmed), "unconfirmed_keys": unconfirmed, "stored": len(stored), + "lost_confirmed": len(confirmed - stored), + "stored_minus_confirmed": len(stored) - len(confirmed), +}, indent=2)) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh index 1b84587..ef623f1 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,5 +1,4 @@ #!/bin/sh set -eu - -docker compose restart postgres redis api - +docker compose restart postgres redis +docker compose restart api lb \ No newline at end of file diff --git a/scripts/rolling-under-load.sh b/scripts/rolling-under-load.sh new file mode 100755 index 0000000..9ab775b --- /dev/null +++ b/scripts/rolling-under-load.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p evidence +docker run --rm --network mtusi2026lab3_edge -v "$PWD":/w python:3.13-alpine \ + python /w/tests/load.py --base-url http://lb:8080 --duration 80 --concurrency 4 \ + --output /w/evidence/load-rolling.json > evidence/load-rolling.txt & +LOAD=$! +sleep 8 +date '+%T rolling start' +./scripts/rolling-update.sh +date '+%T rolling end' +wait "$LOAD" +docker compose ps > evidence/ps-after-rolling.txt +docker compose logs lb --since 4m > evidence/lb-rolling.log +for c in $(docker compose ps -q api); do docker exec "$c" printenv APP_VERSION; done +cat evidence/load-rolling.txt diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..91d24bc --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +N=2 +export APP_VERSION="${APP_VERSION:-v$(date +%s)}" +wait_healthy() { + until [ "$(docker compose ps -q api | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$1" ]; do sleep 1; done +} +docker compose build api +for old in $(docker compose ps -q api); do + docker compose up -d --no-deps --no-recreate --scale api=$((N+1)) api + wait_healthy $((N+1)) + sleep 5 # LB: rise 2 + обновление DNS + echo "$(date +%T) replacing $(docker inspect -f '{{.Name}}' "$old")" + docker stop -t 20 "$old" >/dev/null + docker rm "$old" >/dev/null +done +wait_healthy $N +echo "$(date +%T) done, APP_VERSION=$APP_VERSION" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..daa80f1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +cid="" +for id in $(docker compose ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$host" = "$TARGET_INSTANCE_ID" ]; then cid="$id"; break; fi +done +[ -n "$cid" ] || { echo "container for instance $TARGET_INSTANCE_ID not found" >&2; exit 1; } + +docker stop -t 8 "$cid" >/dev/null +echo "stopped $cid" >&2