feat: API replicas over HAProxy (3)

This commit is contained in:
2026-11-06 03:26:54 +03:00
parent de2f63b926
commit ec111aadfb
32 changed files with 396 additions and 147 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
+17 -1
View File
@@ -2,12 +2,22 @@ PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
COMPOSE ?= podman compose
TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/database_url 2>/dev/null)
TEST_REDIS_URL ?= redis://127.0.0.1:56379/0
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-go
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-go: Go tests against the running stand"
up:
$(COMPOSE) up -d --build
down:
$(COMPOSE) down
check:
$(PYTHON) scripts/check_contract.py
@@ -28,3 +38,9 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/idempotency-race.py
test-go:
@TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./...
+5 -20
View File
@@ -18,10 +18,7 @@ import (
"messenger/internal/service"
)
const (
shutdownTimeout = 5 * time.Second
migrateAttempts = 60
)
const shutdownTimeout = 5 * time.Second
func main() {
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
@@ -37,7 +34,8 @@ func run(log *slog.Logger) error {
return err
}
if err := migrateWithRetry(log, cfg.DatabaseURL); err != nil {
instanceID, err := os.Hostname()
if err != nil {
return err
}
@@ -63,7 +61,7 @@ func run(log *slog.Logger) error {
srv := &http.Server{
Addr: cfg.Addr(),
Handler: router.New(handler.New(users, auth, chats, messages, deps, log)),
Handler: router.New(handler.New(users, auth, chats, messages, deps, log), instanceID),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
@@ -75,7 +73,7 @@ func run(log *slog.Logger) error {
errCh := make(chan error, 1)
go func() {
log.Info("starting server", "addr", srv.Addr, "env", cfg.Env, "session_ttl", cfg.SessionTTL.String())
log.Info("starting server", "addr", srv.Addr, "instance_id", instanceID, "env", cfg.Env, "session_ttl", cfg.SessionTTL.String())
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
errCh <- err
}
@@ -96,16 +94,3 @@ func run(log *slog.Logger) error {
log.Info("server stopped")
return nil
}
func migrateWithRetry(log *slog.Logger, databaseURL string) error {
var err error
for attempt := 1; attempt <= migrateAttempts; attempt++ {
if err = postgres.Migrate(databaseURL); err == nil {
log.Info("migrations applied")
return nil
}
log.Warn("migrations failed, retrying", "attempt", attempt, "err", err)
time.Sleep(time.Second)
}
return err
}
+30 -6
View File
@@ -1,4 +1,4 @@
name: messenger
name: messenger-lab3
services:
postgres:
@@ -34,9 +34,22 @@ services:
retries: 30
restart: unless-stopped
migrate:
image: docker.io/migrate/migrate:v4.20.1
entrypoint: ["/bin/sh", "-c"]
command: ['migrate -path /migrations -database "$$(cat /run/secrets/database_url)" up']
secrets:
- database_url
volumes:
- ./internal/repository/postgres/migrations:/migrations:ro
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
image: localhost/messenger-api:lab2
image: localhost/messenger-api:lab3
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
@@ -47,13 +60,24 @@ services:
secrets:
- database_url
- cache_url
deploy:
mode: replicated
replicas: ${API_REPLICAS:-2}
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
restart: unless-stopped
lb:
image: docker.io/library/haproxy:3.4.7-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "8080:8080"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
- api
restart: unless-stopped
secrets:
+14
View File
@@ -0,0 +1,14 @@
$ api replicas before
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 10 seconds
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 10 seconds
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 87, "failed_attempts": 1, "first_recovery_seconds": 2.52}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 26 seconds
messenger-lab3_api_2 hostname=2751ae93e6d1 Exited (0) 15 seconds ago
+18
View File
@@ -0,0 +1,18 @@
$ api replicas before (api_2 restored with: podman compose start api)
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 2 minutes
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 19 seconds
$ curl -s localhost:8080/health/live (shift round-robin so the test write lands on the other replica)
{"status":"ok"}
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 81, "failed_attempts": 1, "first_recovery_seconds": 2.424}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
messenger-lab3_api_1 hostname=0405cf8a4a54 Exited (0) 15 seconds ago
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 35 seconds
+8
View File
@@ -0,0 +1,8 @@
BASE_URL="http://localhost:8080" python3 scripts/idempotency-race.py
parallel requests with one key: 20
successful responses (201): 20
failed requests: 0 []
distinct message ids in responses: 1
messages stored in history: 1
instances that served requests: ['0405cf8a4a54', '2751ae93e6d1']
OK: exactly one message created
+18
View File
@@ -0,0 +1,18 @@
20 GET /api/v1/users/me through LB (localhost:8080) with ONE bearer session, no re-login
X-Instance-Id 0405cf8a4a54 responses 10
X-Instance-Id 2751ae93e6d1 responses 10
$ podman ps --filter label=com.docker.compose.service=api --format "{{.Names}} {{.ID}} {{.Status}}"
messenger-lab3_api_1 0405cf8a4a54 Up 54 seconds
messenger-lab3_api_2 2751ae93e6d1 Up 53 seconds
$ podman inspect -f "{{.Name}} hostname={{.Config.Hostname}} ports={{.NetworkSettings.Ports}}" api replicas
messenger-lab3_api_1 hostname=0405cf8a4a54 ports=map[8080/tcp:[]]
messenger-lab3_api_2 hostname=2751ae93e6d1 ports=map[8080/tcp:[]]
$ podman ps --format "{{.Names}} {{.Ports}}" (only lb publishes 8080)
messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp
messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp
messenger-lab3_api_1 8080/tcp
messenger-lab3_api_2 8080/tcp
messenger-lab3_lb_1 0.0.0.0:8080->8080/tcp
+15
View File
@@ -0,0 +1,15 @@
$ podman ps -a --filter label=com.docker.compose.service=migrate
messenger-lab3_migrate_1 Exited (0) 53 seconds ago
$ podman logs messenger-lab3_migrate_1
1/u init (10.119477ms)
2/u message_idempotency (14.878187ms)
no change
no change
no change
$ podman logs messenger-lab3_api_1 | grep -i migrat (API replicas do not run migrations)
0
$ psql: SELECT version, dirty FROM schema_migrations
2|f
+25
View File
@@ -0,0 +1,25 @@
$ containers before
messenger-lab3_postgres_1 8c509b4f5143 Up About a minute (healthy)
messenger-lab3_redis_1 b5db2a433051 Up About a minute (healthy)
messenger-lab3_migrate_1 9fa71320cdba Exited (0) About a minute ago
messenger-lab3_api_1 0405cf8a4a54 Up About a minute
messenger-lab3_api_2 2751ae93e6d1 Up About a minute
messenger-lab3_lb_1 d3a35fe2295d Up About a minute
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ containers after (same IDs, fresh Up time = restarted, volumes kept)
messenger-lab3_postgres_1 8c509b4f5143 Up 1 second (healthy)
messenger-lab3_redis_1 b5db2a433051 Up 1 second (healthy)
messenger-lab3_migrate_1 9fa71320cdba Exited (0) Less than a second ago
messenger-lab3_api_1 0405cf8a4a54 Up Less than a second
messenger-lab3_api_2 2751ae93e6d1 Up Less than a second
messenger-lab3_lb_1 d3a35fe2295d Up Less than a second
$ podman volume ls
messenger-lab3_pgdata
messenger-lab3_redisdata
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+42
View File
@@ -0,0 +1,42 @@
? messenger/cmd/server [no test files]
? messenger/internal/config [no test files]
? messenger/internal/domain [no test files]
=== RUN TestSessionExpiry
--- PASS: TestSessionExpiry (3.09s)
=== RUN TestLogout
--- PASS: TestLogout (0.08s)
=== RUN TestWrongPassword
--- PASS: TestWrongPassword (0.10s)
=== RUN TestRedisUnavailable
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestRedisUnavailable (2.89s)
=== RUN TestDatabaseUnavailable
--- PASS: TestDatabaseUnavailable (0.17s)
=== RUN TestInvalidJSON
--- PASS: TestInvalidJSON (0.01s)
=== RUN TestForeignChat
--- PASS: TestForeignChat (0.17s)
=== RUN TestEmptyMessage
--- PASS: TestEmptyMessage (0.08s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.09s)
=== RUN TestPaginationMultiplePages
--- PASS: TestPaginationMultiplePages (0.08s)
=== RUN TestConcurrentMessages
--- PASS: TestConcurrentMessages (0.52s)
PASS
ok messenger/internal/handler 7.273s
=== RUN TestRegisterUsernameRace
--- PASS: TestRegisterUsernameRace (0.02s)
=== RUN TestCreateChatRollback
--- PASS: TestCreateChatRollback (0.02s)
PASS
ok messenger/internal/repository/postgres 0.043s
? messenger/internal/repository/redis [no test files]
? messenger/internal/router [no test files]
? messenger/internal/service [no test files]
-2
View File
@@ -4,7 +4,6 @@ go 1.27.1
require (
github.com/go-chi/chi/v5 v5.3.2
github.com/golang-migrate/migrate/v4 v4.20.1
github.com/jackc/pgx/v5 v5.11.0
github.com/redis/go-redis/v9 v9.23.0
golang.org/x/crypto v0.58.0
@@ -12,7 +11,6 @@ require (
require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
-56
View File
@@ -1,40 +1,12 @@
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang-migrate/migrate/v4 v4.20.1 h1:2N/ToVTKrKl58ynBpgeVJ4In7VcLCjWTZtm4eP1LxhU=
github.com/golang-migrate/migrate/v4 v4.20.1/go.mod h1:DDPgKVb4ovSWc4FwSPfV2Uz1160f4XBiTHTrAJtljmM=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -45,24 +17,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg=
github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY=
github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
@@ -73,16 +27,6 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+28
View File
@@ -0,0 +1,28 @@
global
log stdout format raw local0 info
defaults
mode http
log global
option httplog
timeout connect 2s
timeout client 30s
timeout server 20s
timeout http-request 10s
retries 3
option redispatch
resolvers dns
parse-resolv-conf
hold valid 1s
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers dns init-addr none
server-template api 4 api:8080
+1 -1
View File
@@ -42,7 +42,7 @@ func (h *Handler) fail(w http.ResponseWriter, r *http.Request, err error) {
case errors.Is(err, domain.ErrNotFound):
writeError(w, http.StatusNotFound, "not_found", "resource not found")
case errors.Is(err, domain.ErrConflict):
writeError(w, http.StatusConflict, "conflict", "username is already taken")
writeError(w, http.StatusConflict, "conflict", err.Error())
case errors.Is(err, domain.ErrUnavailable):
h.log.Error("dependency unavailable", "request_id", w.Header().Get(headerRequestID),
"method", r.Method, "path", r.URL.Path, "err", err)
+4 -1
View File
@@ -57,7 +57,7 @@ func newStack(t *testing.T, dbURL, redisURL string, ttl time.Duration) *httptest
chats := service.NewChatService(store)
messages := service.NewMessageService(chats, store)
h := handler.New(users, auth, chats, messages, []handler.Pinger{store.Ping, sessions.Ping}, slog.New(slog.DiscardHandler))
srv := httptest.NewServer(router.New(h))
srv := httptest.NewServer(router.New(h, "test"))
t.Cleanup(srv.Close)
return srv
}
@@ -81,6 +81,9 @@ func request(t *testing.T, srv *httptest.Server, method, path, token, body strin
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
if method == http.MethodPost && strings.HasSuffix(path, "/messages") {
req.Header.Set("Idempotency-Key", uuid.New().String())
}
resp, err := srv.Client().Do(req)
if err != nil {
t.Fatal(err)
+4 -1
View File
@@ -2,8 +2,11 @@ package handler
import (
"context"
"fmt"
"net/http"
"time"
"messenger/internal/domain"
)
const readyTimeout = time.Second
@@ -17,7 +20,7 @@ func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) {
defer cancel()
for _, dep := range h.deps {
if err := dep(ctx); err != nil {
h.fail(w, r, err)
h.fail(w, r, fmt.Errorf("%w: %v", domain.ErrUnavailable, err))
return
}
}
+14 -2
View File
@@ -1,6 +1,13 @@
package handler
import "net/http"
import (
"net/http"
"regexp"
)
const headerIdempotencyKey = "Idempotency-Key"
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
chatID, err := chatIDParam(r)
@@ -8,6 +15,11 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
h.fail(w, r, err)
return
}
key := r.Header.Get(headerIdempotencyKey)
if !idempotencyKeyRe.MatchString(key) {
h.fail(w, r, badRequest("Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}"))
return
}
var req createMessageRequest
if err := decodeJSON(w, r, &req); err != nil {
h.fail(w, r, err)
@@ -17,7 +29,7 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
h.fail(w, r, badRequest("text is required"))
return
}
msg, err := h.messages.Send(r.Context(), actor(r), chatID, *req.Text)
msg, err := h.messages.Send(r.Context(), actor(r), chatID, key, *req.Text)
if err != nil {
h.fail(w, r, err)
return
+10
View File
@@ -17,11 +17,21 @@ import (
const (
headerRequestID = "X-Request-Id"
headerInstanceID = "X-Instance-Id"
headerWWWAuthenticate = "WWW-Authenticate"
bearerChallenge = `Bearer realm="messenger"`
basicChallenge = `Basic realm="messenger", charset="UTF-8"`
)
func InstanceID(id string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(headerInstanceID, id)
next.ServeHTTP(w, r)
})
}
}
func RequestID(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(headerRequestID, uuid.New().String())
-39
View File
@@ -1,39 +0,0 @@
package postgres
import (
"embed"
"errors"
"strings"
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/pgx/v5"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
//go:embed migrations/*.sql
var migrationsFS embed.FS
func Migrate(databaseURL string) error {
src, err := iofs.New(migrationsFS, "migrations")
if err != nil {
return err
}
m, err := migrate.NewWithSourceInstance("iofs", src, migrateURL(databaseURL))
if err != nil {
return err
}
defer m.Close()
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
return nil
}
func migrateURL(databaseURL string) string {
for _, prefix := range []string{"postgresql://", "postgres://"} {
if rest, ok := strings.CutPrefix(databaseURL, prefix); ok {
return "pgx5://" + rest
}
}
return databaseURL
}
@@ -0,0 +1 @@
DROP TABLE message_idempotency;
@@ -0,0 +1,9 @@
CREATE TABLE message_idempotency (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
key text NOT NULL,
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
@@ -21,9 +21,6 @@ func newStore(t *testing.T) (*postgres.Store, *pgxpool.Pool) {
if dbURL == "" {
t.Skip("TEST_DATABASE_URL is required")
}
if err := postgres.Migrate(dbURL); err != nil {
t.Fatal(err)
}
pool, err := postgres.NewPool(context.Background(), dbURL)
if err != nil {
t.Fatal(err)
+42 -4
View File
@@ -169,11 +169,49 @@ func (s *Store) loadMembers(ctx context.Context, chats []domain.Chat) error {
return dbErr(rows.Err())
}
func (s *Store) CreateMessage(ctx context.Context, m domain.Message) error {
_, err := s.pool.Exec(ctx,
func (s *Store) CreateMessageIdempotent(ctx context.Context, m domain.Message, key, requestHash string) (domain.Message, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return domain.Message{}, dbErr(err)
}
defer tx.Rollback(ctx)
tag, err := tx.Exec(ctx, `
INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT DO NOTHING`,
m.SenderID, m.ChatID, key, requestHash, m.ID, m.CreatedAt)
if err != nil {
return domain.Message{}, dbErr(err)
}
if tag.RowsAffected() == 0 {
var storedHash string
var stored domain.Message
err := tx.QueryRow(ctx, `
SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency k
JOIN messages m ON m.id = k.message_id
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
m.SenderID, m.ChatID, key,
).Scan(&storedHash, &stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
if err != nil {
return domain.Message{}, dbErr(err)
}
if storedHash != requestHash {
return domain.Message{}, domain.ErrConflict
}
return stored, nil
}
if _, err := tx.Exec(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`,
m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt)
return dbErr(err)
m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt); err != nil {
return domain.Message{}, dbErr(err)
}
if err := tx.Commit(ctx); err != nil {
return domain.Message{}, dbErr(err)
}
return m, nil
}
func (s *Store) ListMessages(ctx context.Context, chatID uuid.UUID, after *domain.Cursor, limit int) ([]domain.Message, bool, error) {
+2 -2
View File
@@ -8,9 +8,9 @@ import (
"messenger/internal/handler"
)
func New(h *handler.Handler) http.Handler {
func New(h *handler.Handler, instanceID string) http.Handler {
r := chi.NewRouter()
r.Use(handler.RequestID, h.AccessLog, h.Recoverer)
r.Use(handler.InstanceID(instanceID), handler.RequestID, h.AccessLog, h.Recoverer)
r.NotFound(h.NotFound)
r.MethodNotAllowed(h.MethodNotAllowed)
+9 -4
View File
@@ -2,6 +2,9 @@ package service
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"uuid"
@@ -17,7 +20,7 @@ func NewMessageService(chats *ChatService, messages MessageRepository) *MessageS
return &MessageService{chats: chats, messages: messages}
}
func (s *MessageService) Send(ctx context.Context, actor domain.User, chatID uuid.UUID, text string) (domain.Message, error) {
func (s *MessageService) Send(ctx context.Context, actor domain.User, chatID uuid.UUID, key, text string) (domain.Message, error) {
if _, err := s.chats.Get(ctx, actor, chatID); err != nil {
return domain.Message{}, err
}
@@ -31,10 +34,12 @@ func (s *MessageService) Send(ctx context.Context, actor domain.User, chatID uui
Text: text,
CreatedAt: now(),
}
if err := s.messages.CreateMessage(ctx, m); err != nil {
return domain.Message{}, err
sum := sha256.Sum256([]byte(text))
stored, err := s.messages.CreateMessageIdempotent(ctx, m, key, hex.EncodeToString(sum[:]))
if errors.Is(err, domain.ErrConflict) {
return domain.Message{}, fmt.Errorf("%w: Idempotency-Key was already used with a different body", domain.ErrConflict)
}
return m, nil
return stored, err
}
func (s *MessageService) History(ctx context.Context, actor domain.User, chatID uuid.UUID, after *domain.Cursor, limit int) (domain.Page[domain.Message], error) {
+1 -1
View File
@@ -21,7 +21,7 @@ type ChatRepository interface {
}
type MessageRepository interface {
CreateMessage(ctx context.Context, m domain.Message) error
CreateMessageIdempotent(ctx context.Context, m domain.Message, key, requestHash string) (domain.Message, error)
ListMessages(ctx context.Context, chatID uuid.UUID, after *domain.Cursor, limit int) ([]domain.Message, bool, error)
}
+4
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"errors"
"fmt"
"regexp"
"unicode/utf8"
@@ -49,6 +50,9 @@ func (s *UserService) Register(ctx context.Context, username, displayName, passw
CreatedAt: now(),
}
if err := s.repo.CreateUser(ctx, u, hash); err != nil {
if errors.Is(err, domain.ErrConflict) {
return domain.User{}, fmt.Errorf("%w: username is already taken", domain.ErrConflict)
}
return domain.User{}, err
}
return u, nil
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
import sys
import threading
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
WORKERS = 20
def main():
c = Client()
alice, _, _, chat = c.bootstrap()
key = uuid.uuid4().hex
text = 'Одновременный повтор ' + key
barrier = threading.Barrier(WORKERS)
results, errors = [], []
lock = threading.Lock()
def send():
barrier.wait()
try:
r = c.message(alice, chat, text, key=key)
with lock:
results.append((r['json'], r['headers']['X-Instance-Id']))
except AssertionError as exc:
with lock:
errors.append(str(exc))
threads = [threading.Thread(target=send) for _ in range(WORKERS)]
for t in threads:
t.start()
for t in threads:
t.join()
history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items']
stored = [m for m in history if m['text'] == text]
instances = sorted({i for _, i in results})
ids = {m['id'] for m, _ in results}
print('parallel requests with one key:', WORKERS)
print('successful responses (201):', len(results))
print('failed requests:', len(errors), errors[:3])
print('distinct message ids in responses:', len(ids))
print('messages stored in history:', len(stored))
print('instances that served requests:', instances)
check(not errors, 'some requests failed')
check(len(ids) == 1 and len(stored) == 1, 'duplicate message created')
check(all(m == results[0][0] for m, _ in results), 'replayed responses differ')
check(len(instances) >= 2, 'requests were not spread across replicas')
print('OK: exactly one message created')
if __name__ == '__main__':
main()
+3 -3
View File
@@ -1,7 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
COMPOSE=${COMPOSE:-docker compose}
COMPOSE=${COMPOSE:-podman compose}
$COMPOSE stop api postgres redis
$COMPOSE start postgres redis api
$COMPOSE stop lb api redis postgres
$COMPOSE start postgres redis api lb
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
ENGINE=${ENGINE:-podman}
PROJECT=${PROJECT:-messenger-lab3}
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for id in $($ENGINE ps -q --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api"); do
if [ "$($ENGINE inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
$ENGINE stop -t 5 "$id" >/dev/null
exit 0
fi
done
exit 1