This commit is contained in:
2026-10-22 18:41:56 +03:00
parent 665c37c0b8
commit 3695ab6dfa
32 changed files with 881 additions and 156 deletions
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
secrets/
+25 -7
View File
@@ -4,7 +4,9 @@ import (
"context"
"log"
"net/http"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
@@ -66,30 +68,46 @@ func main() {
chatCache := cache.New(redisClient, chatCacheTTL)
s := store.New(pool, limiter)
mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache)
instanceID := config.GetEnv("INSTANCE_ID", "")
if instanceID == "" {
instanceID, _ = os.Hostname()
}
var draining atomic.Bool
mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache, httpapi.Options{
InstanceID: instanceID,
Redis: redisClient,
Draining: &draining,
})
port := config.GetEnv("PORT", "8080")
srv := &http.Server{
Addr: ":" + port,
Handler: mux,
Addr: ":" + port,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
IdleTimeout: 60 * time.Second,
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
go func() {
log.Printf("запуск на :%s", port)
log.Printf("запуск на :%s instance=%s", port, instanceID)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("сервер упал: %v", err)
}
}()
<-ctx.Done()
log.Println("Получен сигнал остановки, завершаю активные запросы...")
log.Println("Получен сигнал остановки: readiness -> 503, жду LB...")
draining.Store(true)
time.Sleep(time.Duration(config.ParseIntEnv("DRAIN_SECONDS", 4)) * time.Second)
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
shutdownCtx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
log.Printf("принудительное завершение: %v", err)
}
+52 -11
View File
@@ -3,8 +3,10 @@ services:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres-data:/var/lib/postgresql/data
healthcheck:
@@ -13,8 +15,10 @@ services:
timeout: 3s
retries: 10
start_period: 5s
ports:
- "127.0.0.1:5431:5432"
networks: [data]
deploy:
resources:
limits: { cpus: "1.0", memory: 512M }
redis:
image: redis:7.4-alpine
@@ -27,6 +31,10 @@ services:
timeout: 3s
retries: 10
start_period: 5s
networks: [data]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
migrate:
image: migrate/migrate:v4.17.1
@@ -40,6 +48,7 @@ services:
- "-database=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable"
- "up"
restart: "no"
networks: [data]
api:
build:
@@ -54,23 +63,55 @@ services:
condition: service_healthy
environment:
PORT: ${PORT}
DATABASE_URL: ${DATABASE_URL}
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
MAX_MESSAGES: ${MAX_MESSAGES}
MAX_CONCURRENT_REQUESTS: ${MAX_CONCURRENT_REQUESTS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_CONN_TIMEOUT_SECONDS: ${DB_CONN_TIMEOUT_SECONDS}
CACHE_URL: ${CACHE_URL}
CHAT_CACHE_TTL_SECONDS: 30
ports:
- "127.0.0.1:8080:8080"
DRAIN_SECONDS: 4
GOMEMLIMIT: 200MiB
APP_VERSION: ${APP_VERSION:-dev}
secrets:
- database_url
stop_grace_period: 20s
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
interval: 10s
timeout: 3s
retries: 5
start_period: 10s
interval: 3s
timeout: 2s
retries: 3
start_period: 5s
networks: [edge, data]
deploy:
replicas: 2
resources:
limits: { cpus: "0.5", memory: 256M }
lb:
image: haproxy:3.0-alpine
depends_on:
api:
condition: service_healthy
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:8080:8080"
networks: [edge]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
networks:
edge:
data:
secrets:
database_url:
file: ./secrets/database_url.txt
postgres_password:
file: ./secrets/postgres_password.txt
volumes:
postgres-data:
+16
View File
@@ -0,0 +1,16 @@
{
"container": "mtusi2026lab3-api-5",
"stopped_instance": "ba50cd9c50eb",
"rps_before": 891.5,
"rps_during": 850.6,
"rps_after": 966.7,
"first_attempt_errors": 0,
"error_window_s_after_stop": null,
"longest_gap_without_success_s": 0.18,
"stopped_instance_seen_while_down": false,
"confirmed": 31607,
"unconfirmed_keys": 0,
"stored": 31607,
"lost_confirmed": 0,
"stored_minus_confirmed": 0
}
+16
View File
@@ -0,0 +1,16 @@
{
"container": "mtusi2026lab3-api-6",
"stopped_instance": "f9ec7883c11a",
"rps_before": 978.8,
"rps_during": 975.9,
"rps_after": 961.9,
"first_attempt_errors": 0,
"error_window_s_after_stop": null,
"longest_gap_without_success_s": 0.09,
"stopped_instance_seen_while_down": false,
"confirmed": 33997,
"unconfirmed_keys": 0,
"stored": 33997,
"lost_confirmed": 0,
"stored_minus_confirmed": 0
}
+20
View File
@@ -0,0 +1,20 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-5 mtusi2026lab3-api "./api" api 4 minutes ago Up About a minute (healthy) 8080/tcp
mtusi2026lab3-api-6 mtusi2026lab3-api "./api" api 4 minutes ago Up 32 seconds (healthy) 8080/tcp
mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 4 minutes ago Up About a minute 127.0.0.1:8080->8080/tcp
mtusi2026lab3-migrate-1 migrate/migrate:v4.17.1 "migrate -path=/migr…" migrate 11 hours ago Exited (0) 32 seconds ago
mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up About a minute (healthy) 5432/tcp
mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 4 minutes ago Up About a minute (healthy) 6379/tcp
/mtusi2026lab3-api-5 cpus=500000000 mem=268435456
/mtusi2026lab3-api-6 cpus=500000000 mem=268435456
/mtusi2026lab3-lb-1 cpus=250000000 mem=134217728
/mtusi2026lab3-postgres-1 cpus=1000000000 mem=536870912
/mtusi2026lab3-redis-1 cpus=250000000 mem=134217728
0
migrate-1 | no change
migrate-1 | no change
migrate-1 | no change
migrate-1 | no change
migrate-1 | no change
migrate-1 | no change
DATABASE_URL_FILE=/run/secrets/database_url
+8
View File
@@ -0,0 +1,8 @@
PASS one token served by >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a']
PASS concurrent same key: all 201 [201]
PASS concurrent same key: one message id 1
PASS concurrent requests hit >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a']
PASS sequential repeat: same id
PASS sequential repeat: >=2 replicas ['ba50cd9c50eb', 'f9ec7883c11a']
PASS different body, same key -> 409
PASS history has exactly 2 messages 2
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.041,
"concurrency": 4,
"successful_requests": 28040,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 933.404,
"successful_latency_ms": {
"p50": 2.346,
"p95": 7.173,
"p99": 48.063
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 31.868,
"concurrency": 4,
"successful_requests": 2491,
"failed_requests": 20,
"error_ratio": 0.007965,
"successful_rps": 78.166,
"successful_latency_ms": {
"p50": 3.893,
"p95": 8.14,
"p99": 18.909
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 80.0,
"elapsed_seconds": 80.043,
"concurrency": 4,
"successful_requests": 55307,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 690.963,
"successful_latency_ms": {
"p50": 2.553,
"p95": 24.852,
"p99": 51.029
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 80.0,
"elapsed_seconds": 80.043,
"concurrency": 4,
"successful_requests": 55307,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 690.963,
"successful_latency_ms": {
"p50": 2.553,
"p95": 24.852,
"p99": 51.029
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 31.564,
"concurrency": 4,
"successful_requests": 2495,
"failed_requests": 19,
"error_ratio": 0.007558,
"successful_rps": 79.047,
"successful_latency_ms": {
"p50": 3.615,
"p95": 6.022,
"p99": 13.593
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 31.564,
"concurrency": 4,
"successful_requests": 2495,
"failed_requests": 19,
"error_ratio": 0.007558,
"successful_rps": 79.047,
"successful_latency_ms": {
"p50": 3.615,
"p95": 6.022,
"p99": 13.593
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+6
View File
@@ -0,0 +1,6 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-7 mtusi2026lab3-api "./api" api 40 seconds ago Up 39 seconds (healthy) 8080/tcp
mtusi2026lab3-api-8 mtusi2026lab3-api "./api" api 24 seconds ago Up 23 seconds (healthy) 8080/tcp
mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 8 minutes ago Up 5 minutes 127.0.0.1:8080->8080/tcp
mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 8 minutes ago Up 5 minutes (healthy) 5432/tcp
mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 8 minutes ago Up 5 minutes (healthy) 6379/tcp
+5
View File
@@ -0,0 +1,5 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-1 mtusi2026lab3-api "./api" api 14 minutes ago Up 7 minutes (healthy) 8080/tcp
mtusi2026lab3-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 21 minutes ago Up 14 minutes 127.0.0.1:8080->8080/tcp
mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 21 minutes ago Up 14 minutes (healthy) 127.0.0.1:5431->5432/tcp
mtusi2026lab3-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 21 minutes ago Up 14 minutes (healthy) 6379/tcp
+31
View File
@@ -0,0 +1,31 @@
global
log stdout format raw local0
defaults
mode http
timeout connect 2s
timeout client 15s
timeout server 15s
retries 2
option redispatch
log global
option httplog
timeout check 2s
resolvers docker
nameserver dns 127.0.0.11:53
hold valid 2s
timeout resolve 1s
timeout retry 1s
resolve_retries 3
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-6 api:8080
+6
View File
@@ -3,6 +3,7 @@ package config
import (
"os"
"strconv"
"strings"
"time"
)
@@ -10,6 +11,11 @@ func GetEnv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
if p := os.Getenv(key + "_FILE"); p != "" {
if b, err := os.ReadFile(p); err == nil {
return strings.TrimSpace(string(b))
}
}
return fallback
}
+45 -14
View File
@@ -12,6 +12,7 @@ import (
"time"
"mtusi2026/internal/cache"
"mtusi2026/internal/idgen"
"mtusi2026/internal/model"
"mtusi2026/internal/sessionstore"
"mtusi2026/internal/store"
@@ -21,6 +22,39 @@ import (
//вспомогательные функции
func newKey() string { return idgen.GenerateUUIDv4() }
func doMessage(srv *httptest.Server, chatID, token, key string, body any) (*http.Response, error) {
var buf bytes.Buffer
if s, ok := body.(string); ok {
buf.WriteString(s)
} else if err := json.NewEncoder(&buf).Encode(body); err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodPost,
fmt.Sprintf("%s/api/v1/chats/%s/messages", srv.URL, chatID), &buf)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
if key != "" {
req.Header.Set("Idempotency-Key", key)
}
return srv.Client().Do(req)
}
func postMessage(t *testing.T, srv *httptest.Server, chatID, token, key string, body any) *http.Response {
t.Helper()
resp, err := doMessage(srv, chatID, token, key, body)
if err != nil {
t.Fatalf("do request: %v", err)
}
return resp
}
func newUnlimitedLimiter() *store.Limiter {
return store.NewLimiter(1_000_000, 1000)
}
@@ -412,9 +446,8 @@ func TestCreateMessage_EmptyText_Returns400(t *testing.T) {
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{
Text: "",
})
resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: ""})
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected 400 for empty text, got %d", resp.StatusCode)
}
@@ -427,9 +460,8 @@ func TestCreateMessage_WhitespaceOnlyText_Returns400(t *testing.T) {
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{
Text: " \t\n ",
})
resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: " \t\n "})
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected 400 for whitespace-only text, got %d", resp.StatusCode)
}
@@ -445,9 +477,8 @@ func TestCreateMessage_ForeignChat_Returns404(t *testing.T) {
chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID, bob.ID})
resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), carol.Token, model.CreateMessageRequest{
Text: "Я тут чужая",
})
resp := postMessage(t, srv, chat.ID, carol.Token, newKey(), model.CreateMessageRequest{Text: "Я тут чужая"})
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected 404 for foreign chat, got %d", resp.StatusCode)
}
@@ -462,7 +493,7 @@ func TestCreateMessage_SenderIDIgnoredFromRequestBody(t *testing.T) {
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID, bob.ID})
raw := fmt.Sprintf(`{"text":"Привет","sender_id":%q}`, bob.ID)
resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, raw)
resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), raw)
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected 400 (unknown field sender_id), got %d", resp.StatusCode)
}
@@ -479,7 +510,7 @@ func TestListMessages_PaginationAcrossMultiplePages(t *testing.T) {
const total = 5
for i := 0; i < total; i++ {
resp := postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token, model.CreateMessageRequest{
resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{
Text: fmt.Sprintf("сообщение %d", i),
})
if resp.StatusCode != http.StatusCreated {
@@ -536,8 +567,8 @@ func TestListMessages_CursorFromDifferentChat_Returns400(t *testing.T) {
chatA := mustCreateChat(t, srv, alice, "Чат A", []string{alice.ID})
chatB := mustCreateChat(t, srv, alice, "Чат B", []string{alice.ID})
postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chatA.ID), alice.Token, model.CreateMessageRequest{Text: "первое"}).Body.Close()
postJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chatA.ID), alice.Token, model.CreateMessageRequest{Text: "второе"}).Body.Close()
postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "первое"}).Body.Close()
postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "второе"}).Body.Close()
respA := getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages?limit=1", chatA.ID), alice.Token)
pageA := decodeBody[model.MessagePage](t, respA)
@@ -761,7 +792,7 @@ func TestChatCache_WarmCache(t *testing.T) {
[]string{user.ID},
)
// Первый GET прогревает cache.
// Первый GET прогревает cache
resp := getJSON(
t,
srv,
+13 -1
View File
@@ -3,18 +3,24 @@ package httpapi
import (
"context"
"net/http"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
func handleHealthLive(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, newRequestID(), map[string]string{"status": "ok"})
}
func handleHealthReady(pool *pgxpool.Pool) http.HandlerFunc {
func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.Bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
if draining.Load() {
writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "shutting down")
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
@@ -22,6 +28,12 @@ func handleHealthReady(pool *pgxpool.Pool) http.HandlerFunc {
writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable")
return
}
if rdb != nil {
if err := rdb.Ping(ctx).Err(); err != nil {
writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "session store unavailable")
return
}
}
writeJSON(w, http.StatusOK, requestID, map[string]string{"status": "ok"})
}
}
+184
View File
@@ -0,0 +1,184 @@
package httpapi
import (
"fmt"
"net/http"
"strings"
"sync"
"testing"
"mtusi2026/internal/model"
)
func TestIdempotency_RepeatReturnsSameMessage(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
key := newKey()
body := model.CreateMessageRequest{Text: "привет"}
r1 := postMessage(t, srv, chat.ID, alice.Token, key, body)
if r1.StatusCode != http.StatusCreated {
t.Fatalf("first: expected 201, got %d", r1.StatusCode)
}
m1 := decodeBody[model.Message](t, r1)
r2 := postMessage(t, srv, chat.ID, alice.Token, key, body)
if r2.StatusCode != http.StatusCreated {
t.Fatalf("repeat: expected 201, got %d", r2.StatusCode)
}
m2 := decodeBody[model.Message](t, r2)
if m1.ID != m2.ID || !m1.CreatedAt.Equal(m2.CreatedAt) {
t.Fatalf("repeat returned a different message: %+v vs %+v", m1, m2)
}
page := decodeBody[model.MessagePage](t,
getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token))
if len(page.Items) != 1 {
t.Fatalf("expected 1 message in history, got %d", len(page.Items))
}
}
func TestIdempotency_DifferentBodySameKey_Returns409(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
key := newKey()
postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "один"}).Body.Close()
resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "два"})
defer resp.Body.Close()
if resp.StatusCode != http.StatusConflict {
t.Fatalf("expected 409, got %d", resp.StatusCode)
}
}
func TestIdempotency_MissingOrInvalidKey_Returns400(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
keys := map[string]string{
"missing": "",
"space": "bad key",
"too long": strings.Repeat("a", 129),
"cyrillic": "ключ",
}
for name, key := range keys {
resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "x"})
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("%s: expected 400, got %d", name, resp.StatusCode)
}
}
}
func TestIdempotency_ConcurrentSameKey_OneMessage(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID})
const workers = 20
key := newKey()
start := make(chan struct{})
type result struct {
status int
id string
err error
}
results := make(chan result, workers)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
resp, err := doMessage(srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "гонка"})
if err != nil {
results <- result{err: err}
return
}
defer resp.Body.Close()
var m model.Message
_ = t.Fatal
results <- result{status: resp.StatusCode, id: m.ID}
}()
}
close(start)
wg.Wait()
close(results)
ids := map[string]bool{}
for r := range results {
if r.err != nil {
t.Errorf("request error: %v", r.err)
continue
}
if r.status != http.StatusCreated {
t.Errorf("expected 201, got %d", r.status)
}
ids[r.id] = true
}
if len(ids) != 1 {
t.Fatalf("expected one distinct message id, got %d", len(ids))
}
page := decodeBody[model.MessagePage](t,
getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token))
if len(page.Items) != 1 {
t.Fatalf("expected exactly 1 message stored, got %d", len(page.Items))
}
}
func TestIdempotency_ScopeIsPerUserAndChat(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
bob := mustRegisterUser(t, srv, "bob")
chatA := mustCreateChat(t, srv, alice, "A", []string{alice.ID, bob.ID})
chatB := mustCreateChat(t, srv, alice, "B", []string{alice.ID, bob.ID})
key := newKey()
body := model.CreateMessageRequest{Text: "то же самое"}
ids := map[string]bool{}
for _, c := range []struct {
chat string
token string
}{
{chatA.ID, alice.Token}, {chatB.ID, alice.Token}, {chatA.ID, bob.Token},
} {
resp := postMessage(t, srv, c.chat, c.token, key, body)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected 201, got %d", resp.StatusCode)
}
ids[decodeBody[model.Message](t, resp).ID] = true
}
if len(ids) != 3 {
t.Fatalf("same key in different scopes must create 3 messages, got %d", len(ids))
}
}
func TestIdempotency_ForeignChatRepeat_Returns404(t *testing.T) {
srv := newTestServer(t)
defer srv.Close()
alice := mustRegisterUser(t, srv, "alice")
carol := mustRegisterUser(t, srv, "carol")
chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID})
key := newKey()
for i := 0; i < 2; i++ {
resp := postMessage(t, srv, chat.ID, carol.Token, key, model.CreateMessageRequest{Text: "чужой"})
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("attempt %d: expected 404, got %d", i, resp.StatusCode)
}
}
}
+16 -7
View File
@@ -1,36 +1,45 @@
package httpapi
import (
"errors"
"net/http"
"regexp"
"mtusi2026/internal/model"
"mtusi2026/internal/store"
)
var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u model.User, requestID string) {
chatID := r.PathValue("chat_id")
key := r.Header.Get("Idempotency-Key")
if !idemKeyRe.MatchString(key) {
writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "missing or invalid Idempotency-Key")
return
}
var req model.CreateMessageRequest
if err := decodeJSONStrict(r, &req); err != nil {
writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "malformed request body")
return
}
m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, req.Text)
m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text)
if err != nil {
switch err {
case store.ErrChatNotFound:
switch {
case errors.Is(err, store.ErrChatNotFound):
writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found")
case store.ErrCapacityExceeded:
writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "message storage capacity reached")
case store.ErrValidation:
case errors.Is(err, store.ErrIdempotencyConflict):
writeError(w, http.StatusConflict, requestID, codeConflict, "Idempotency-Key was used with a different request body")
case errors.Is(err, store.ErrValidation):
writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid text")
default:
writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error")
}
return
}
writeJSON(w, http.StatusCreated, requestID, m)
}
+1
View File
@@ -44,6 +44,7 @@ func TestConcurrentMessages_NoLostOrDuplicated(t *testing.T) {
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+alice.Token)
req.Header.Set("Idempotency-Key", fmt.Sprintf("race-%d-%d", clientNum, i))
resp, err := client.Do(req)
if err != nil {
errCh <- fmt.Errorf("client %d: do request: %w", clientNum, err)
+47 -47
View File
@@ -2,8 +2,10 @@ package httpapi
import (
"net/http"
"sync/atomic"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
"mtusi2026/internal/cache"
"mtusi2026/internal/model"
@@ -11,40 +13,18 @@ import (
"mtusi2026/internal/store"
)
type Options struct {
InstanceID string
Redis *redis.Client
Draining *atomic.Bool
}
func NewRouter(s *store.Store, pool *pgxpool.Pool, sessionStores ...sessionstore.Store) http.Handler {
var ss sessionstore.Store
if len(sessionStores) > 0 {
ss = sessionStores[0]
}
a := &api{
store: s,
sessionStore: ss,
}
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", handleHealthLive)
mux.HandleFunc("GET /health/ready", handleHealthReady(pool))
mux.HandleFunc("POST /api/v1/users", a.handleRegisterUser)
mux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser))
mux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat))
mux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats))
mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat))
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage))
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages))
mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession)
mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware(func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) {
a.handleDeleteSession(w, r)
}))
var handler http.Handler = mux
handler = loggingMiddleware(handler)
handler = concurrencyMiddleware(s.Limiter, handler)
return handler
return NewRouterWithCache(s, pool, ss, nil)
}
func NewRouterWithCache(
@@ -52,33 +32,53 @@ func NewRouterWithCache(
pool *pgxpool.Pool,
sessionStore sessionstore.Store,
chatCache *cache.Cache,
opts ...Options,
) http.Handler {
var o Options
if len(opts) > 0 {
o = opts[0]
}
if o.InstanceID == "" {
o.InstanceID = "local"
}
if o.Draining == nil {
o.Draining = new(atomic.Bool)
}
a := &api{
store: s,
sessionStore: sessionStore,
chatCache: chatCache,
}
apiMux := http.NewServeMux()
apiMux.HandleFunc("POST /api/v1/users", a.handleRegisterUser)
apiMux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser))
apiMux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat))
apiMux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats))
apiMux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat))
apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage))
apiMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages))
apiMux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession)
apiMux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware(
func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) {
a.handleDeleteSession(w, r)
}))
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", handleHealthLive)
mux.HandleFunc("GET /health/ready", handleHealthReady(pool))
mux.HandleFunc("POST /api/v1/users", a.handleRegisterUser)
mux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser))
mux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat))
mux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats))
mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat))
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage))
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages))
mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession)
mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware(func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) {
a.handleDeleteSession(w, r)
}))
root := http.NewServeMux()
root.HandleFunc("GET /health/live", handleHealthLive)
root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Draining))
root.Handle("/", concurrencyMiddleware(s.Limiter, apiMux))
var handler http.Handler = mux
handler = loggingMiddleware(handler)
handler = concurrencyMiddleware(s.Limiter, handler)
var h http.Handler = root
h = loggingMiddleware(h)
h = instanceMiddleware(o.InstanceID, h)
return h
}
return handler
func instanceMiddleware(id string, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", id)
next.ServeHTTP(w, r)
})
}
+51 -66
View File
@@ -2,11 +2,12 @@ package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"strconv"
"sync/atomic"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5"
@@ -14,6 +15,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"mtusi2026/internal/cursor"
"mtusi2026/internal/idgen"
"mtusi2026/internal/model"
)
@@ -233,56 +235,75 @@ func (s *Store) ListChatsForUser(ctx context.Context, userID, cursorStr string,
return page, next, nil
}
// --- Сообщения ---
func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, text string) (model.Message, error) {
if s.Limiter != nil {
if !s.Limiter.TryReserveMessage() {
return model.Message{}, ErrCapacityExceeded
}
}
release := func() {
if s.Limiter != nil {
s.Limiter.ReleaseMessage()
}
}
var ErrIdempotencyConflict = errors.New("idempotency key reused with different body")
func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string) (model.Message, error) {
if !validMessageText(text) {
release()
return model.Message{}, ErrValidation
}
sum := sha256.Sum256([]byte(text))
hash := hex.EncodeToString(sum[:])
tx, err := s.pool.Begin(ctx)
if err != nil {
return model.Message{}, err
}
defer tx.Rollback(ctx)
var isMember bool
err := s.pool.QueryRow(ctx,
`SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
chatID, senderID,
).Scan(&isMember)
if err != nil {
release()
if err := tx.QueryRow(ctx,
`SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`,
chatID, senderID).Scan(&isMember); err != nil {
return model.Message{}, err
}
if !isMember {
release()
return model.Message{}, ErrChatNotFound
}
msgID := idgen.GenerateUUIDv4()
tag, err := tx.Exec(ctx,
`INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id)
VALUES ($1,$2,$3,$4,$5)
ON CONFLICT (user_id, chat_id, key) DO NOTHING`,
senderID, chatID, idemKey, hash, msgID)
if err != nil {
return model.Message{}, err
}
if tag.RowsAffected() == 0 {
var storedHash string
var m model.Message
err := tx.QueryRow(ctx,
`SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`,
senderID, chatID, idemKey,
).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
if err != nil {
return model.Message{}, err
}
if storedHash != hash {
return model.Message{}, ErrIdempotencyConflict
}
return m, nil
}
var m model.Message
err = s.pool.QueryRow(ctx,
`INSERT INTO messages (chat_id, sender_id, text)
VALUES ($1, $2, $3)
err = tx.QueryRow(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1,$2,$3,$4, clock_timestamp())
RETURNING id, chat_id, sender_id, text, created_at`,
chatID, senderID, text,
msgID, chatID, senderID, text,
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
if err != nil {
release()
if isCheckViolation(err) {
return model.Message{}, ErrValidation
}
return model.Message{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.Message{}, err
}
return m, nil
}
@@ -377,44 +398,8 @@ func validMessageText(s string) bool {
return false
}
func (s *Store) CreateSession(ctx context.Context, token, userID string, ttl time.Duration) (time.Time, error) {
expiresAt := time.Now().UTC().Add(ttl)
_, err := s.pool.Exec(ctx,
`INSERT INTO sessions (token, user_id, expires_at) VALUES ($1, $2, $3)`,
token, userID, expiresAt,
)
if err != nil {
return time.Time{}, err
}
return expiresAt, nil
}
var ErrSessionNotFound = errors.New("session not found or expired")
func (s *Store) GetUserBySessionToken(ctx context.Context, token string) (model.User, error) {
var u model.User
err := s.pool.QueryRow(ctx,
`SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions s
JOIN users u ON u.id = s.user_id
WHERE s.token = $1 AND s.expires_at > now()`,
token,
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt)
if err != nil {
return model.User{}, ErrSessionNotFound
}
return u, nil
}
func (s *Store) DeleteSession(ctx context.Context, token string) (bool, error) {
tag, err := s.pool.Exec(ctx, `DELETE FROM sessions WHERE token = $1`, token)
if err != nil {
return false, err
}
return tag.RowsAffected() > 0, nil
}
func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.User, string, error) {
var u model.User
var passwordHash string
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idx_idempotency_created_at ON idempotency_keys (created_at);
+65
View File
@@ -0,0 +1,65 @@
import base64, json, sys, urllib.error, urllib.request, uuid
from concurrent.futures import ThreadPoolExecutor
BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080"
PWD = "correct-horse-battery-staple"
def call(method, path, body=None, token=None, basic=None, key=None):
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(BASE + path, data=data, method=method)
if body is not None:
req.add_header("Content-Type", "application/json")
if token:
req.add_header("Authorization", "Bearer " + token)
if basic:
req.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode())
if key:
req.add_header("Idempotency-Key", key)
try:
with urllib.request.urlopen(req, timeout=10) as r:
raw = r.read()
return r.status, r.headers.get("X-Instance-Id"), json.loads(raw) if raw else None
except urllib.error.HTTPError as e:
return e.code, e.headers.get("X-Instance-Id"), None
ok = True
def check(name, cond, info=""):
global ok
ok &= bool(cond)
print(("PASS" if cond else "FAIL"), name, info)
name = "lb_" + uuid.uuid4().hex[:10]
s, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "LB check", "password": PWD})
s, _, sess = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}")
token = sess["token"]
# один токен разные реплики
inst = {call("GET", "/api/v1/users/me", token=token)[1] for _ in range(12)}
check("one token served by >=2 replicas", len(inst) >= 2, sorted(inst))
s, _, chat = call("POST", "/api/v1/chats", {"title": "lb", "member_ids": [user["id"]]}, token=token)
path = f"/api/v1/chats/{chat['id']}/messages"
# конкурентные повторы с одним ключом через lb
key = "conc-" + uuid.uuid4().hex
with ThreadPoolExecutor(30) as ex:
res = list(ex.map(lambda _: call("POST", path, {"text": "race"}, token=token, key=key), range(30)))
ids = {r[2]["id"] for r in res if r[2]}
check("concurrent same key: all 201", all(r[0] == 201 for r in res), sorted({r[0] for r in res}))
check("concurrent same key: one message id", len(ids) == 1, len(ids))
check("concurrent requests hit >=2 replicas", len({r[1] for r in res}) >= 2, sorted({r[1] for r in res}))
# последовательный повтор на разных репликах
key2 = "seq-" + uuid.uuid4().hex
seq = [call("POST", path, {"text": "seq"}, token=token, key=key2) for _ in range(6)]
check("sequential repeat: same id", len({r[2]["id"] for r in seq}) == 1)
check("sequential repeat: >=2 replicas", len({r[1] for r in seq}) >= 2, sorted({r[1] for r in seq}))
check("different body, same key -> 409", call("POST", path, {"text": "other"}, token=token, key=key2)[0] == 409)
s, _, page = call("GET", path + "?limit=100", token=token)
check("history has exactly 2 messages", len(page["items"]) == 2, len(page["items"]))
sys.exit(0 if ok else 1)
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
mkdir -p evidence
python3 tests/load.py --duration 30 --concurrency 4 \
--output evidence/load-stop-api2.json > evidence/load-stop-api2.txt &
LOAD_PID=$!
sleep 8
date '+%T stop api-2'
docker stop -t 8 mtusi2026lab3-api-2 >/dev/null
docker compose ps > evidence/ps-during-failover.txt
sleep 10
date '+%T start api-2'
docker start mtusi2026lab3-api-2 >/dev/null
wait "$LOAD_PID"
docker compose logs lb --since 3m > evidence/lb-failover.log
cat evidence/load-stop-api2.txt
+91
View File
@@ -0,0 +1,91 @@
import base64, json, subprocess, sys, threading, time, urllib.request, uuid
import os
BASE = os.environ.get("BASE", "http://localhost:8080"); PWD = "correct-horse-battery-staple"
WORKERS, DURATION, STOP_AT, START_AT = 4, 35, 8, 22
def req(method, path, body=None, token=None, basic=None, key=None):
r = urllib.request.Request(BASE + path, method=method,
data=json.dumps(body).encode() if body is not None else None)
if body is not None: r.add_header("Content-Type", "application/json")
if token: r.add_header("Authorization", "Bearer " + token)
if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode())
if key: r.add_header("Idempotency-Key", key)
with urllib.request.urlopen(r, timeout=3) as resp:
return resp.status, resp.headers.get("X-Instance-Id"), json.loads(resp.read() or b"null")
def sh(*a): return subprocess.run(a, capture_output=True, text=True).stdout.split()
arg = sys.argv[1]
names = sorted(sh("docker", "ps", "--filter", "label=com.docker.compose.service=api",
"--format", "{{.Names}}"))
container = names[int(arg) - 1] if arg.isdigit() else arg
stopped_id = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", container],
capture_output=True, text=True).stdout.strip()
assert stopped_id, "container not found"
name = "fo_" + uuid.uuid4().hex[:10]
_, _, u = req("POST", "/api/v1/users", {"username": name, "display_name": "fo", "password": PWD})
_, _, s = req("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}")
tok = s["token"]
_, _, c = req("POST", "/api/v1/chats", {"title": "fo", "member_ids": [u["id"]]}, token=tok)
path = f"/api/v1/chats/{c['id']}/messages"
t0 = time.time()
attempts, confirmed, unconfirmed = [], set(), 0
lock = threading.Lock()
def action():
time.sleep(STOP_AT)
subprocess.run(["docker", "stop", "-t", "8", container], capture_output=True)
time.sleep(max(0, START_AT - (time.time() - t0)))
subprocess.run(["docker", "start", container], capture_output=True)
def worker():
global unconfirmed
while time.time() - t0 < DURATION:
key = uuid.uuid4().hex
done = False
for n in range(3):
ts = time.time() - t0
try:
_, inst, m = req("POST", path, {"text": "x"}, token=tok, key=key)
with lock:
attempts.append((ts, True, inst, n)); confirmed.add(m["id"])
done = True
break
except Exception:
with lock: attempts.append((ts, False, None, n))
time.sleep(0.2)
if not done:
with lock: unconfirmed += 1
threads = [threading.Thread(target=action)] + [threading.Thread(target=worker) for _ in range(WORKERS)]
[t.start() for t in threads]; [t.join() for t in threads]
stored, cur = set(), None
while True:
_, _, page = req("GET", path + "?limit=100" + (f"&cursor={cur}" if cur else ""), token=tok)
stored |= {m["id"] for m in page["items"]}
cur = page["next_cursor"]
if not cur: break
oks = sorted(a[0] for a in attempts if a[1])
gap = max((b - a for a, b in zip(oks, oks[1:])), default=0)
first_fail = [a[0] - STOP_AT for a in attempts if not a[1] and a[3] == 0 and a[0] >= STOP_AT]
pre_stop_errors = sum(1 for a in attempts if not a[1] and a[3] == 0 and a[0] < STOP_AT)
rps = lambda a, b: round(sum(a <= t < b for t in oks) / (b - a), 1)
late = {a[2] for a in attempts if a[1] and STOP_AT + 6 < a[0] < START_AT}
print(json.dumps({
"container": container, "stopped_instance": stopped_id,
"rps_before": rps(0, STOP_AT), "rps_during": rps(STOP_AT, START_AT), "rps_after": rps(START_AT, DURATION),
"first_attempt_errors": len(first_fail),
"error_window_s_after_stop": [round(min(first_fail), 2), round(max(first_fail), 2)] if first_fail else None,
"longest_gap_without_success_s": round(gap, 2),
"stopped_instance_seen_while_down": stopped_id in late,
"confirmed": len(confirmed), "unconfirmed_keys": unconfirmed, "stored": len(stored),
"lost_confirmed": len(confirmed - stored),
"stored_minus_confirmed": len(stored) - len(confirmed),
}, indent=2))
+2 -3
View File
@@ -1,5 +1,4 @@
#!/bin/sh
set -eu
docker compose restart postgres redis api
docker compose restart postgres redis
docker compose restart api lb
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
mkdir -p evidence
docker run --rm --network mtusi2026lab3_edge -v "$PWD":/w python:3.13-alpine \
python /w/tests/load.py --base-url http://lb:8080 --duration 80 --concurrency 4 \
--output /w/evidence/load-rolling.json > evidence/load-rolling.txt &
LOAD=$!
sleep 8
date '+%T rolling start'
./scripts/rolling-update.sh
date '+%T rolling end'
wait "$LOAD"
docker compose ps > evidence/ps-after-rolling.txt
docker compose logs lb --since 4m > evidence/lb-rolling.log
for c in $(docker compose ps -q api); do docker exec "$c" printenv APP_VERSION; done
cat evidence/load-rolling.txt
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
N=2
export APP_VERSION="${APP_VERSION:-v$(date +%s)}"
wait_healthy() {
until [ "$(docker compose ps -q api | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$1" ]; do sleep 1; done
}
docker compose build api
for old in $(docker compose ps -q api); do
docker compose up -d --no-deps --no-recreate --scale api=$((N+1)) api
wait_healthy $((N+1))
sleep 5 # LB: rise 2 + обновление DNS
echo "$(date +%T) replacing $(docker inspect -f '{{.Name}}' "$old")"
docker stop -t 20 "$old" >/dev/null
docker rm "$old" >/dev/null
done
wait_healthy $N
echo "$(date +%T) done, APP_VERSION=$APP_VERSION"
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cid=""
for id in $(docker compose ps -q api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then cid="$id"; break; fi
done
[ -n "$cid" ] || { echo "container for instance $TARGET_INSTANCE_ID not found" >&2; exit 1; }
docker stop -t 8 "$cid" >/dev/null
echo "stopped $cid" >&2