From 7fa8881eb329b5c43d0601a3abe05f8564589c3b Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 16:58:11 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20X-Instance-Id,=20=D0=B7=D0=B0=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=BA=D1=83=20drain=20=D0=BF=D1=80=D0=B8=20SIGTERM=20=D0=B8=20?= =?UTF-8?q?=D0=BE=D1=82=D0=B4=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=BF=D1=83=D1=81=D0=BA=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 13 ++-- cmd/migrate/main.go | 58 ++++++++++++++ cmd/server/main.go | 39 +++++----- compose.yaml | 17 +++++ internal/app/app.go | 11 ++- internal/app/app_test.go | 7 ++ internal/app/auth_test.go | 7 +- internal/app/failure_test.go | 1 + internal/platform/config/config.go | 83 ++++++++++++++++++--- internal/platform/config/config_test.go | 58 ++++++++++++-- internal/platform/httpx/engine.go | 7 +- internal/platform/httpx/httpx_test.go | 23 +++++- internal/platform/httpx/middleware.go | 9 +++ internal/platform/ident/ident.go | 11 +++ internal/platform/ident/ident_test.go | 12 +++ internal/platform/postgres/postgres.go | 10 +-- internal/platform/postgres/postgres_test.go | 15 +++- internal/server/server.go | 19 +++-- internal/server/server_test.go | 49 +++++++++++- internal/testenv/testenv.go | 2 +- tools/concurrency/main_test.go | 2 +- 21 files changed, 389 insertions(+), 64 deletions(-) create mode 100644 cmd/migrate/main.go diff --git a/Dockerfile b/Dockerfile index b8bc70b1..571f6bb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,23 @@ -# Build stage: compile a static binary with the exact toolchain pinned in go.mod. +# Build stage: compile static binaries with the exact toolchain pinned in go.mod. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download && go mod verify COPY . . -RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server +RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Runtime stage: minimal image, unprivileged user, the server is PID 1. +# Runtime stage: minimal image, unprivileged user, the server is PID 1. The +# same image runs the one-shot migrate service with its own entrypoint. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app -COPY --from=build /out/server /usr/local/bin/server +COPY --from=build /out/server /out/migrate /usr/local/bin/ USER app ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab EXPOSE 8080 -HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 \ +# The short start interval lets `compose up --wait` and the rolling update see +# a new replica as healthy about a second after it is ready. +HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 ENTRYPOINT ["/usr/local/bin/server"] diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..201cec5d --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,58 @@ +// Command migrate applies the embedded schema migrations once and exits. It +// runs as a one-shot service before the API replicas start, so the replicas +// never race each other over the schema and need no rights to change it. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" +) + +// poolSize covers the goose session lock and the migration itself. +const poolSize = 2 + +func main() { + cfg, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + log := newLogger(cfg) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg); err != nil { + log.Error("migration failed", "error", err.Error()) + os.Exit(1) + } +} + +func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, poolSize) + if err != nil { + return err + } + defer pool.Close() + applied, err := postgres.Migrate(ctx, pool) + if err != nil { + return err + } + log.Info("migrations applied", "applied_migrations", applied) + return nil +} + +func newLogger(cfg config.Migrate) *slog.Logger { + opts := &slog.HandlerOptions{Level: cfg.LogLevel} + if cfg.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 305e4ff2..6964a810 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -8,10 +8,12 @@ import ( "net" "os" "os/signal" + "runtime" "syscall" "messenger/internal/app" "messenger/internal/platform/config" + "messenger/internal/platform/ident" "messenger/internal/platform/postgres" "messenger/internal/platform/valkey" "messenger/internal/server" @@ -23,30 +25,29 @@ func main() { fmt.Fprintln(os.Stderr, "configuration error:", err) os.Exit(2) } - log := newLogger(cfg) + // Every log line carries the alias that this process sends as + // X-Instance-Id, so a response can be matched to its replica's log. + alias := ident.NewAlias() + log := newLogger(cfg).With("instance", alias) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() - if err := run(ctx, log, cfg); err != nil { + if err := run(ctx, log, cfg, alias); err != nil { log.Error("server stopped with error", "error", err.Error()) os.Exit(1) } } // run connects the dependencies before it listens, so the port accepts -// connections only once the schema is in place. -func run(ctx context.Context, log *slog.Logger, cfg config.Config) error { - pool, err := postgres.NewPool(ctx, cfg.DatabaseURL) +// connections only once they answer. The schema is applied beforehand by the +// separate migrate command, once per deployment rather than by every replica. +func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns) if err != nil { return err } defer pool.Close() - applied, err := postgres.Migrate(ctx, pool) - if err != nil { - return err - } - log.Info("schema ready", "applied_migrations", applied) sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL) if err != nil { return err @@ -58,15 +59,19 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config) error { return fmt.Errorf("listen on %s: %w", cfg.Addr(), err) } log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(), - "shutdown_timeout", cfg.ShutdownTimeout.String(), "session_ttl", cfg.SessionTTL.String()) + "gomaxprocs", runtime.GOMAXPROCS(0), "db_max_conns", cfg.DBMaxConns, + "drain_delay", cfg.DrainDelay.String(), "shutdown_timeout", cfg.ShutdownTimeout.String(), + "session_ttl", cfg.SessionTTL.String(), "idempotency_ttl", cfg.IdempotencyTTL.String()) a := app.New(app.Deps{ - Log: log, - DB: pool, - Sessions: sessions, - SessionTTL: cfg.SessionTTL, - MaxBodyBytes: cfg.MaxBodyBytes, + Log: log, + DB: pool, + Sessions: sessions, + SessionTTL: cfg.SessionTTL, + MaxBodyBytes: cfg.MaxBodyBytes, + InstanceID: alias, + IdempotencyTTL: cfg.IdempotencyTTL, }) - return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.ShutdownTimeout) + return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) } func newLogger(cfg config.Config) *slog.Logger { diff --git a/compose.yaml b/compose.yaml index 0a11d1f6..9228acc9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,8 @@ services: - database_url - session_store_url depends_on: + migrate: + condition: service_completed_successfully postgres: condition: service_healthy valkey: @@ -28,6 +30,21 @@ services: - "${PUBLISH:-127.0.0.1:8080}:8080" stop_grace_period: 10s + # One-shot: applies the schema before the API starts, then exits. + migrate: + image: mtusi-messenger:lab3 + entrypoint: ["/usr/local/bin/migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 environment: diff --git a/internal/app/app.go b/internal/app/app.go index 6653c8c5..288122a2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -25,6 +25,8 @@ import ( type App struct { Engine *gin.Engine Health *health.Handler + // Chats is exposed for the background cleanup of idempotency keys. + Chats *chat.Service } // Deps are the shared resources the modules are built on; their lifetime is @@ -37,6 +39,11 @@ type Deps struct { MaxBodyBytes int64 // RequestTimeout bounds every API request; zero means the default. RequestTimeout time.Duration + // InstanceID is sent as X-Instance-Id on every response. + InstanceID string + // IdempotencyTTL is how long a message Idempotency-Key is remembered; + // zero means the default. + IdempotencyTTL time.Duration } // defaultRequestTimeout stays below the server's 10s write timeout, so a @@ -47,7 +54,7 @@ func New(d Deps) *App { users := user.NewService(userstore.NewUsers(d.DB)) chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users) - engine := httpx.NewEngine(d.Log, d.MaxBodyBytes) + engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) probes := health.New( health.Check{Name: "postgres", Probe: d.DB.Ping}, health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, @@ -66,5 +73,5 @@ func New(d Deps) *App { user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) - return &App{Engine: engine, Health: probes} + return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index b9ba72b6..e908cf83 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -50,6 +50,9 @@ func TestMain(m *testing.M) { const testPassword = "correct-horse-battery" +// testInstance is the alias of the in-process application under test. +const testInstance = "00000000000000a1" + // errorBody mirrors the contract error envelope. type errorBody struct { Error struct { @@ -103,6 +106,7 @@ func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester { Sessions: kv.Client, SessionTTL: ttl, MaxBodyBytes: 64 * 1024, + InstanceID: testInstance, })} } @@ -135,6 +139,9 @@ func (ts *tester) do(method, path, token string, body any, headers map[string]st if res.requestID == "" { ts.t.Fatalf("%s %s: missing X-Request-Id", method, path) } + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + ts.t.Fatalf("%s %s: X-Instance-Id = %q, want %q", method, path, got, testInstance) + } if res.status >= 400 { var eb errorBody if err := json.Unmarshal(res.raw, &eb); err != nil { diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go index 602d1e2a..727cd6a3 100644 --- a/internal/app/auth_test.go +++ b/internal/app/auth_test.go @@ -146,11 +146,14 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") } +// The TTL leaves room for a loaded machine: with one second the first request +// after login sometimes arrived too late under -race with all packages +// running at once. func TestSessionExpiresAfterTTL(t *testing.T) { - ts := newTesterWithTTL(t, time.Second) + ts := newTesterWithTTL(t, 3*time.Second) alice := ts.user("alice", "A") ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") - time.Sleep(1200 * time.Millisecond) + time.Sleep(3200 * time.Millisecond) ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized") } diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go index cadc4ee6..155c8000 100644 --- a/internal/app/failure_test.go +++ b/internal/app/failure_test.go @@ -35,6 +35,7 @@ func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *te SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, RequestTimeout: requestTimeout, + InstanceID: testInstance, }) return &tester{t: t, a: a}, pg, vk } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 30100b71..0dcc0d9d 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -25,6 +25,22 @@ type Config struct { DatabaseURL string SessionStoreURL string SessionTTL time.Duration + // DrainDelay is how long a stopping process keeps serving with readiness + // already off, so the load balancer can take it out of rotation before + // the listener closes. It must exceed the balancer's detection time. + DrainDelay time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + // DebugAddr enables the internal expvar listener; empty disables it. + DebugAddr string +} + +// Migrate is the configuration of the one-shot migration command, which +// needs the database only. +type Migrate struct { + Env string + LogLevel slog.Level + DatabaseURL string } // Lookup reads one variable, reporting whether it was set; os.LookupEnv @@ -40,6 +56,9 @@ func Load(lookup Lookup) (Config, error) { ShutdownTimeout: 8 * time.Second, MaxBodyBytes: 64 * 1024, SessionTTL: time.Hour, + DrainDelay: 2 * time.Second, + IdempotencyTTL: 24 * time.Hour, + DBMaxConns: 10, } if v, ok := lookup("APP_HOST"); ok { if strings.TrimSpace(v) == "" { @@ -54,15 +73,9 @@ func Load(lookup Lookup) (Config, error) { } cfg.Port = port } - if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { - cfg.Env = v - } - if v, ok := lookup("APP_LOG_LEVEL"); ok { - var level slog.Level - if err := level.UnmarshalText([]byte(v)); err != nil { - return Config{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) - } - cfg.LogLevel = level + var err error + if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + return Config{}, err } if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { d, err := time.ParseDuration(v) @@ -85,7 +98,33 @@ func Load(lookup Lookup) (Config, error) { } cfg.SessionTTL = time.Duration(n) * time.Second } - var err error + if v, ok := lookup("APP_DRAIN_DELAY"); ok { + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return Config{}, fmt.Errorf("APP_DRAIN_DELAY must be a non-negative duration such as 2s, got %q", v) + } + cfg.DrainDelay = d + } + if v, ok := lookup("IDEMPOTENCY_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.IdempotencyTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("DB_MAX_CONNS"); ok { + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 || n > 1000 { + return Config{}, fmt.Errorf("DB_MAX_CONNS must be an integer between 1 and 1000, got %q", v) + } + cfg.DBMaxConns = int32(n) + } + if v, ok := lookup("APP_DEBUG_ADDR"); ok && v != "" { + if _, _, err := net.SplitHostPort(v); err != nil { + return Config{}, fmt.Errorf("APP_DEBUG_ADDR must be host:port such as :9090, got %q", v) + } + cfg.DebugAddr = v + } if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { return Config{}, err } @@ -95,6 +134,30 @@ func Load(lookup Lookup) (Config, error) { return cfg, nil } +func LoadMigrate(lookup Lookup) (Migrate, error) { + cfg := Migrate{Env: "lab", LogLevel: slog.LevelInfo} + var err error + if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + return Migrate{}, err + } + if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + return Migrate{}, err + } + return cfg, nil +} + +func logging(lookup Lookup, env string, level slog.Level) (string, slog.Level, error) { + if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { + env = v + } + if v, ok := lookup("APP_LOG_LEVEL"); ok { + if err := level.UnmarshalText([]byte(v)); err != nil { + return "", 0, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + } + } + return env, level, nil +} + // secret reads the file named by the variable; surrounding whitespace, // including the trailing newline editors add, is dropped. func secret(lookup Lookup, key string) (string, error) { diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go index 2288dcd2..4900ff3d 100644 --- a/internal/platform/config/config_test.go +++ b/internal/platform/config/config_test.go @@ -65,6 +65,9 @@ func TestDefaults(t *testing.T) { if cfg.SessionTTL != time.Hour { t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL) } + if cfg.DrainDelay != 2*time.Second || cfg.IdempotencyTTL != 24*time.Hour || cfg.DBMaxConns != 10 || cfg.DebugAddr != "" { + t.Fatalf("lab3 defaults = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } } func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { @@ -92,13 +95,17 @@ func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { func TestOverrides(t *testing.T) { cfg, err := config.Load(env(required(t, map[string]string{ - "APP_HOST": "127.0.0.1", - "APP_PORT": "9090", - "APP_ENV": "dev", - "APP_LOG_LEVEL": "debug", - "APP_SHUTDOWN_TIMEOUT": "3s", - "APP_MAX_BODY_BYTES": "1024", - "SESSION_TTL_SECONDS": "300", + "APP_HOST": "127.0.0.1", + "APP_PORT": "9090", + "APP_ENV": "dev", + "APP_LOG_LEVEL": "debug", + "APP_SHUTDOWN_TIMEOUT": "3s", + "APP_MAX_BODY_BYTES": "1024", + "SESSION_TTL_SECONDS": "300", + "APP_DRAIN_DELAY": "0s", + "IDEMPOTENCY_TTL_SECONDS": "60", + "DB_MAX_CONNS": "25", + "APP_DEBUG_ADDR": ":9090", }))) if err != nil { t.Fatalf("Load error: %v", err) @@ -109,6 +116,9 @@ func TestOverrides(t *testing.T) { if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { t.Fatalf("cfg = %+v", cfg) } + if cfg.DrainDelay != 0 || cfg.IdempotencyTTL != time.Minute || cfg.DBMaxConns != 25 || cfg.DebugAddr != ":9090" { + t.Fatalf("lab3 overrides = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } } func TestInvalidValuesAreReported(t *testing.T) { @@ -126,10 +136,42 @@ func TestInvalidValuesAreReported(t *testing.T) { "ttl negative": {"SESSION_TTL_SECONDS": "-5"}, "ttl text": {"SESSION_TTL_SECONDS": "hour"}, "ttl fraction": {"SESSION_TTL_SECONDS": "1.5"}, + "drain negative": {"APP_DRAIN_DELAY": "-1s"}, + "drain text": {"APP_DRAIN_DELAY": "abc"}, + "key ttl zero": {"IDEMPOTENCY_TTL_SECONDS": "0"}, + "key ttl text": {"IDEMPOTENCY_TTL_SECONDS": "day"}, + "pool zero": {"DB_MAX_CONNS": "0"}, + "pool too big": {"DB_MAX_CONNS": "1001"}, + "pool text": {"DB_MAX_CONNS": "many"}, + "debug addr": {"APP_DEBUG_ADDR": "nonsense"}, } for name, values := range cases { - if _, err := config.Load(env(required(t, values))); err == nil { + _, err := config.Load(env(required(t, values))) + if err == nil { t.Errorf("%s: want error", name) + continue + } + for key := range values { + if !strings.Contains(err.Error(), key) { + t.Errorf("%s: error %q does not name %s", name, err, key) + } } } } + +func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) { + cfg, err := config.LoadMigrate(env(map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "APP_ENV": "dev", + "APP_LOG_LEVEL": "warn", + })) + if err != nil { + t.Fatalf("LoadMigrate error: %v", err) + } + if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.LogLevel != slog.LevelWarn { + t.Fatalf("cfg = %+v", cfg) + } + if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") { + t.Fatalf("missing database url: err = %v", err) + } +} diff --git a/internal/platform/httpx/engine.go b/internal/platform/httpx/engine.go index ba56a897..4f1f89ad 100644 --- a/internal/platform/httpx/engine.go +++ b/internal/platform/httpx/engine.go @@ -20,8 +20,9 @@ const ( ) // NewEngine builds an engine whose every response, including routing -// failures and panics, carries X-Request-Id and the contract error body. -func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine { +// failures and panics, carries X-Instance-Id, X-Request-Id and the contract +// error body. +func NewEngine(log *slog.Logger, maxBodyBytes int64, instanceID string) *gin.Engine { gin.SetMode(gin.ReleaseMode) e := gin.New() // Redirects would answer with text/plain and without a request id. @@ -29,7 +30,7 @@ func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine { e.RedirectFixedPath = false e.HandleMethodNotAllowed = true - e.Use(requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) + e.Use(instance(instanceID), requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) }) e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") }) return e diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go index a745d5a9..66244ad7 100644 --- a/internal/platform/httpx/httpx_test.go +++ b/internal/platform/httpx/httpx_test.go @@ -33,6 +33,8 @@ type payload struct { Count int `json:"count"` } +const testInstance = "0123456789abcdef" + func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine { t.Helper() return newEngineAt(t, logs, slog.LevelDebug) @@ -44,7 +46,7 @@ func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine logs = &bytes.Buffer{} } log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level})) - e := httpx.NewEngine(log, 64) + e := httpx.NewEngine(log, 64, testInstance) e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": ""}) }) e.GET("/panic", func(c *gin.Context) { panic("secret detail") }) e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) }) @@ -106,6 +108,25 @@ func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, sta } } +// The balancer spreads one client's requests over the replicas; the header +// is how a client tells which process answered, so no response may lack it. +func TestEveryResponseCarriesInstanceID(t *testing.T) { + e := newEngine(t, nil) + for _, r := range []struct{ method, path string }{ + {http.MethodGet, "/ok"}, + {http.MethodGet, "/missing"}, + {http.MethodDelete, "/ok"}, + {http.MethodGet, "/panic"}, + {http.MethodGet, "/fail"}, + {http.MethodPost, "/decode"}, + } { + rec, _ := do(t, e, r.method, r.path, "", nil) + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + t.Errorf("%s %s (status %d): X-Instance-Id = %q, want %q", r.method, r.path, rec.Code, got, testInstance) + } + } +} + func TestUnknownRouteIsJSON404(t *testing.T) { rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil) assertError(t, rec, eb, http.StatusNotFound, "not_found") diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index d153a980..a0a9cc6d 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -13,6 +13,15 @@ import ( "github.com/gin-gonic/gin" ) +// instance names the process that answered, so a client behind the load +// balancer can tell the replicas apart. +func instance(id string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("X-Instance-Id", id) + c.Next() + } +} + // requestID assigns a fresh identifier to every request. // // An incoming X-Request-Id is deliberately not reused. This service is diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go index 121500d9..b70efbe9 100644 --- a/internal/platform/ident/ident.go +++ b/internal/platform/ident/ident.go @@ -7,6 +7,8 @@ package ident import ( + "crypto/rand" + "encoding/hex" "errors" "time" @@ -25,6 +27,15 @@ func New() uuid.UUID { return id } +// NewAlias names one server process for X-Instance-Id. It is random rather +// than the hostname, so the header reveals nothing about the infrastructure, +// and a restarted process gets a new one. +func NewAlias() string { + var raw [8]byte + _, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24. + return hex.EncodeToString(raw[:]) +} + // TimeOf extracts the creation time carried by a UUIDv7 as UTC with // millisecond precision. func TimeOf(id uuid.UUID) time.Time { diff --git a/internal/platform/ident/ident_test.go b/internal/platform/ident/ident_test.go index 1ae020df..d0a57d4d 100644 --- a/internal/platform/ident/ident_test.go +++ b/internal/platform/ident/ident_test.go @@ -2,6 +2,7 @@ package ident_test import ( "bytes" + "regexp" "testing" "time" @@ -70,3 +71,14 @@ func TestParseRejectsNonCanonicalForms(t *testing.T) { } } } + +func TestNewAliasIs16HexAndRandom(t *testing.T) { + a, b := ident.NewAlias(), ident.NewAlias() + hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`) + if !hex16.MatchString(a) || !hex16.MatchString(b) { + t.Fatalf("aliases %q, %q are not 16 lowercase hex characters", a, b) + } + if a == b { + t.Fatalf("two aliases are equal: %q", a) + } +} diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go index 616caf62..6409553e 100644 --- a/internal/platform/postgres/postgres.go +++ b/internal/platform/postgres/postgres.go @@ -1,8 +1,9 @@ // Package postgres opens the connection pool and applies the schema. // -// Pool size and server-side timeouts are constants rather than settings: they -// bound how long one request can hold a connection, and the request deadline -// of the API is chosen against them. +// Server-side timeouts are constants rather than settings: they bound how +// long one request can hold a connection, and the request deadline of the API +// is chosen against them. The pool size is a setting, because the scaling +// study varies it. package postgres import ( @@ -27,7 +28,6 @@ import ( ) const ( - maxConns = 10 connectTimeout = time.Second statementTimeout = 5 * time.Second idleInTxTimeout = 10 * time.Second @@ -35,7 +35,7 @@ const ( const codeUniqueViolation = "23505" -func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { +func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, error) { cfg, err := pgxpool.ParseConfig(dsn) if err != nil { return nil, fmt.Errorf("parse database url: %w", err) diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go index c3a14873..49dc627a 100644 --- a/internal/platform/postgres/postgres_test.go +++ b/internal/platform/postgres/postgres_test.go @@ -184,8 +184,21 @@ func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { } } +// The pool size is an experiment variable of the scaling study, so it is +// passed in rather than fixed. +func TestNewPoolUsesTheRequestedSize(t *testing.T) { + pool, err := postgres.NewPool(context.Background(), db.DSN, 3) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if got := pool.Config().MaxConns; got != 3 { + t.Fatalf("MaxConns = %d, want 3", got) + } +} + func TestClassifyRealConnectFailure(t *testing.T) { - _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1") + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1", 1) if err == nil { t.Fatal("NewPool to a closed port must fail") } diff --git a/internal/server/server.go b/internal/server/server.go index 8a70e0fe..718c66df 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,6 +1,7 @@ // Package server runs the HTTP listener with the lifecycle the lab -// requires: readiness drops first, the listener stops accepting, in-flight -// requests drain within a bounded timeout, then the process exits. +// requires: readiness drops first, the process keeps serving for a drain +// delay while the load balancer notices, then the listener stops accepting, +// in-flight requests drain within a bounded timeout and the process exits. package server import ( @@ -17,8 +18,15 @@ type Readiness interface { SetReady(ready bool) } -// Run serves until ctx is cancelled, then shuts down within shutdownTimeout. -func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, shutdownTimeout time.Duration) error { +// Run serves until ctx is cancelled, keeps serving for drainDelay with +// readiness off, then shuts down within shutdownTimeout. +// +// The balancer learns about a stopping replica only from its failing +// readiness checks. Closing the listener at once would let it send requests +// to a closed port, or over a kept-alive connection the server is closing, +// until the checks catch up. drainDelay must therefore exceed the balancer's +// detection time. +func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error { srv := &http.Server{ Handler: h, ReadHeaderTimeout: 5 * time.Second, @@ -45,8 +53,9 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, case <-ctx.Done(): } - log.Info("shutdown started", "timeout", shutdownTimeout.String()) + log.Info("shutdown started", "drain_delay", drainDelay.String(), "timeout", shutdownTimeout.String()) ready.SetReady(false) + time.Sleep(drainDelay) drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() if err := srv.Shutdown(drain); err != nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 040f172a..4f447f85 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -19,7 +19,7 @@ import ( // newHandler builds the smallest handler with probes; the lifecycle does not // depend on the business modules or their stores. func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { - e := httpx.NewEngine(log, 1024) + e := httpx.NewEngine(log, 1024, "test") probes := health.New() probes.Register(e) return e, probes @@ -34,7 +34,7 @@ func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() base := "http://" + ln.Addr().String() waitFor(t, base+"/health/ready", http.StatusOK) @@ -62,7 +62,7 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() base := "http://" + ln.Addr().String() waitFor(t, base+"/health/ready", http.StatusOK) @@ -85,6 +85,49 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) { t.Fatal("readiness must be switched off during shutdown") } +// During the drain delay the balancer is still learning that the replica is +// going away: readiness already fails, yet new connections are accepted and +// served. Only after the delay does the listener close. +func TestDrainKeepsServingWhileNotReady(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + const drain = 700 * time.Millisecond + go func() { done <- server.Run(ctx, log, ln, h, probes, drain, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + waitFor(t, base+"/health/ready", http.StatusServiceUnavailable) + // A fresh connection, as the balancer opens for a new request. + fresh := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + resp, err := fresh.Get(base + "/health/live") + if err != nil { + t.Fatalf("new connection during the drain delay: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("liveness during drain = %d, want 200", resp.StatusCode) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop after the drain delay") + } + if _, err := fresh.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after the drain delay") + } +} + func waitFor(t *testing.T, url string, status int) { t.Helper() deadline := time.Now().Add(5 * time.Second) diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index 980d191d..57cf53bd 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -62,7 +62,7 @@ func StartPostgres(ctx context.Context) (*DB, error) { db.Close() return nil, fmt.Errorf("postgres address: %w", err) } - db.Pool, err = postgres.NewPool(ctx, db.DSN) + db.Pool, err = postgres.NewPool(ctx, db.DSN, 10) if err != nil { db.Close() return nil, err diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go index eb607c3d..7c9158b4 100644 --- a/tools/concurrency/main_test.go +++ b/tools/concurrency/main_test.go @@ -39,7 +39,7 @@ func TestMain(m *testing.M) { func TestRunAgainstLiveServer(t *testing.T) { log := slog.New(slog.NewTextHandler(io.Discard, nil)) - a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024}) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) srv := httptest.NewServer(a.Engine) defer srv.Close()