Добавить X-Instance-Id, задержку drain при SIGTERM и отдельный запуск миграций

This commit is contained in:
2026-11-02 16:58:11 +03:00
parent 5b9f377783
commit 7fa8881eb3
21 changed files with 389 additions and 64 deletions
+8 -5
View File
@@ -1,20 +1,23 @@
# Build stage: compile a static binary with the exact toolchain pinned in go.mod.
# Build stage: compile static binaries with the exact toolchain pinned in go.mod.
FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download && go mod verify
COPY . .
RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate
# Runtime stage: minimal image, unprivileged user, the server is PID 1.
# Runtime stage: minimal image, unprivileged user, the server is PID 1. The
# same image runs the one-shot migrate service with its own entrypoint.
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
RUN adduser -D -H -u 10001 app
COPY --from=build /out/server /usr/local/bin/server
COPY --from=build /out/server /out/migrate /usr/local/bin/
USER app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 \
# The short start interval lets `compose up --wait` and the rolling update see
# a new replica as healthy about a second after it is ready.
HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \
CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1
ENTRYPOINT ["/usr/local/bin/server"]
+58
View File
@@ -0,0 +1,58 @@
// Command migrate applies the embedded schema migrations once and exits. It
// runs as a one-shot service before the API replicas start, so the replicas
// never race each other over the schema and need no rights to change it.
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"messenger/internal/platform/config"
"messenger/internal/platform/postgres"
)
// poolSize covers the goose session lock and the migration itself.
const poolSize = 2
func main() {
cfg, err := config.LoadMigrate(os.LookupEnv)
if err != nil {
fmt.Fprintln(os.Stderr, "configuration error:", err)
os.Exit(2)
}
log := newLogger(cfg)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
if err := run(ctx, log, cfg); err != nil {
log.Error("migration failed", "error", err.Error())
os.Exit(1)
}
}
func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error {
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, poolSize)
if err != nil {
return err
}
defer pool.Close()
applied, err := postgres.Migrate(ctx, pool)
if err != nil {
return err
}
log.Info("migrations applied", "applied_migrations", applied)
return nil
}
func newLogger(cfg config.Migrate) *slog.Logger {
opts := &slog.HandlerOptions{Level: cfg.LogLevel}
if cfg.Env == "dev" {
return slog.New(slog.NewTextHandler(os.Stdout, opts))
}
return slog.New(slog.NewJSONHandler(os.Stdout, opts))
}
+22 -17
View File
@@ -8,10 +8,12 @@ import (
"net"
"os"
"os/signal"
"runtime"
"syscall"
"messenger/internal/app"
"messenger/internal/platform/config"
"messenger/internal/platform/ident"
"messenger/internal/platform/postgres"
"messenger/internal/platform/valkey"
"messenger/internal/server"
@@ -23,30 +25,29 @@ func main() {
fmt.Fprintln(os.Stderr, "configuration error:", err)
os.Exit(2)
}
log := newLogger(cfg)
// Every log line carries the alias that this process sends as
// X-Instance-Id, so a response can be matched to its replica's log.
alias := ident.NewAlias()
log := newLogger(cfg).With("instance", alias)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
if err := run(ctx, log, cfg); err != nil {
if err := run(ctx, log, cfg, alias); err != nil {
log.Error("server stopped with error", "error", err.Error())
os.Exit(1)
}
}
// run connects the dependencies before it listens, so the port accepts
// connections only once the schema is in place.
func run(ctx context.Context, log *slog.Logger, cfg config.Config) error {
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL)
// connections only once they answer. The schema is applied beforehand by the
// separate migrate command, once per deployment rather than by every replica.
func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error {
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
if err != nil {
return err
}
defer pool.Close()
applied, err := postgres.Migrate(ctx, pool)
if err != nil {
return err
}
log.Info("schema ready", "applied_migrations", applied)
sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL)
if err != nil {
return err
@@ -58,15 +59,19 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config) error {
return fmt.Errorf("listen on %s: %w", cfg.Addr(), err)
}
log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(),
"shutdown_timeout", cfg.ShutdownTimeout.String(), "session_ttl", cfg.SessionTTL.String())
"gomaxprocs", runtime.GOMAXPROCS(0), "db_max_conns", cfg.DBMaxConns,
"drain_delay", cfg.DrainDelay.String(), "shutdown_timeout", cfg.ShutdownTimeout.String(),
"session_ttl", cfg.SessionTTL.String(), "idempotency_ttl", cfg.IdempotencyTTL.String())
a := app.New(app.Deps{
Log: log,
DB: pool,
Sessions: sessions,
SessionTTL: cfg.SessionTTL,
MaxBodyBytes: cfg.MaxBodyBytes,
Log: log,
DB: pool,
Sessions: sessions,
SessionTTL: cfg.SessionTTL,
MaxBodyBytes: cfg.MaxBodyBytes,
InstanceID: alias,
IdempotencyTTL: cfg.IdempotencyTTL,
})
return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.ShutdownTimeout)
return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout)
}
func newLogger(cfg config.Config) *slog.Logger {
+17
View File
@@ -19,6 +19,8 @@ services:
- database_url
- session_store_url
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
valkey:
@@ -28,6 +30,21 @@ services:
- "${PUBLISH:-127.0.0.1:8080}:8080"
stop_grace_period: 10s
# One-shot: applies the schema before the API starts, then exits.
migrate:
image: mtusi-messenger:lab3
entrypoint: ["/usr/local/bin/migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
restart: "no"
healthcheck:
disable: true
postgres:
image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873
environment:
+9 -2
View File
@@ -25,6 +25,8 @@ import (
type App struct {
Engine *gin.Engine
Health *health.Handler
// Chats is exposed for the background cleanup of idempotency keys.
Chats *chat.Service
}
// Deps are the shared resources the modules are built on; their lifetime is
@@ -37,6 +39,11 @@ type Deps struct {
MaxBodyBytes int64
// RequestTimeout bounds every API request; zero means the default.
RequestTimeout time.Duration
// InstanceID is sent as X-Instance-Id on every response.
InstanceID string
// IdempotencyTTL is how long a message Idempotency-Key is remembered;
// zero means the default.
IdempotencyTTL time.Duration
}
// defaultRequestTimeout stays below the server's 10s write timeout, so a
@@ -47,7 +54,7 @@ func New(d Deps) *App {
users := user.NewService(userstore.NewUsers(d.DB))
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
probes := health.New(
health.Check{Name: "postgres", Probe: d.DB.Ping},
health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }},
@@ -66,5 +73,5 @@ func New(d Deps) *App {
user.NewHandler(users).Register(public, protected)
chat.NewHandler(chats).Register(protected)
return &App{Engine: engine, Health: probes}
return &App{Engine: engine, Health: probes, Chats: chats}
}
+7
View File
@@ -50,6 +50,9 @@ func TestMain(m *testing.M) {
const testPassword = "correct-horse-battery"
// testInstance is the alias of the in-process application under test.
const testInstance = "00000000000000a1"
// errorBody mirrors the contract error envelope.
type errorBody struct {
Error struct {
@@ -103,6 +106,7 @@ func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester {
Sessions: kv.Client,
SessionTTL: ttl,
MaxBodyBytes: 64 * 1024,
InstanceID: testInstance,
})}
}
@@ -135,6 +139,9 @@ func (ts *tester) do(method, path, token string, body any, headers map[string]st
if res.requestID == "" {
ts.t.Fatalf("%s %s: missing X-Request-Id", method, path)
}
if got := rec.Header().Get("X-Instance-Id"); got != testInstance {
ts.t.Fatalf("%s %s: X-Instance-Id = %q, want %q", method, path, got, testInstance)
}
if res.status >= 400 {
var eb errorBody
if err := json.Unmarshal(res.raw, &eb); err != nil {
+5 -2
View File
@@ -146,11 +146,14 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) {
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "")
}
// The TTL leaves room for a loaded machine: with one second the first request
// after login sometimes arrived too late under -race with all packages
// running at once.
func TestSessionExpiresAfterTTL(t *testing.T) {
ts := newTesterWithTTL(t, time.Second)
ts := newTesterWithTTL(t, 3*time.Second)
alice := ts.user("alice", "A")
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "")
time.Sleep(1200 * time.Millisecond)
time.Sleep(3200 * time.Millisecond)
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized")
}
+1
View File
@@ -35,6 +35,7 @@ func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *te
SessionTTL: time.Hour,
MaxBodyBytes: 64 * 1024,
RequestTimeout: requestTimeout,
InstanceID: testInstance,
})
return &tester{t: t, a: a}, pg, vk
}
+73 -10
View File
@@ -25,6 +25,22 @@ type Config struct {
DatabaseURL string
SessionStoreURL string
SessionTTL time.Duration
// DrainDelay is how long a stopping process keeps serving with readiness
// already off, so the load balancer can take it out of rotation before
// the listener closes. It must exceed the balancer's detection time.
DrainDelay time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
// DebugAddr enables the internal expvar listener; empty disables it.
DebugAddr string
}
// Migrate is the configuration of the one-shot migration command, which
// needs the database only.
type Migrate struct {
Env string
LogLevel slog.Level
DatabaseURL string
}
// Lookup reads one variable, reporting whether it was set; os.LookupEnv
@@ -40,6 +56,9 @@ func Load(lookup Lookup) (Config, error) {
ShutdownTimeout: 8 * time.Second,
MaxBodyBytes: 64 * 1024,
SessionTTL: time.Hour,
DrainDelay: 2 * time.Second,
IdempotencyTTL: 24 * time.Hour,
DBMaxConns: 10,
}
if v, ok := lookup("APP_HOST"); ok {
if strings.TrimSpace(v) == "" {
@@ -54,15 +73,9 @@ func Load(lookup Lookup) (Config, error) {
}
cfg.Port = port
}
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
cfg.Env = v
}
if v, ok := lookup("APP_LOG_LEVEL"); ok {
var level slog.Level
if err := level.UnmarshalText([]byte(v)); err != nil {
return Config{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v)
}
cfg.LogLevel = level
var err error
if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil {
return Config{}, err
}
if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok {
d, err := time.ParseDuration(v)
@@ -85,7 +98,33 @@ func Load(lookup Lookup) (Config, error) {
}
cfg.SessionTTL = time.Duration(n) * time.Second
}
var err error
if v, ok := lookup("APP_DRAIN_DELAY"); ok {
d, err := time.ParseDuration(v)
if err != nil || d < 0 {
return Config{}, fmt.Errorf("APP_DRAIN_DELAY must be a non-negative duration such as 2s, got %q", v)
}
cfg.DrainDelay = d
}
if v, ok := lookup("IDEMPOTENCY_TTL_SECONDS"); ok {
n, err := strconv.ParseUint(v, 10, 32)
if err != nil || n == 0 {
return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be a positive whole number of seconds, got %q", v)
}
cfg.IdempotencyTTL = time.Duration(n) * time.Second
}
if v, ok := lookup("DB_MAX_CONNS"); ok {
n, err := strconv.ParseInt(v, 10, 32)
if err != nil || n < 1 || n > 1000 {
return Config{}, fmt.Errorf("DB_MAX_CONNS must be an integer between 1 and 1000, got %q", v)
}
cfg.DBMaxConns = int32(n)
}
if v, ok := lookup("APP_DEBUG_ADDR"); ok && v != "" {
if _, _, err := net.SplitHostPort(v); err != nil {
return Config{}, fmt.Errorf("APP_DEBUG_ADDR must be host:port such as :9090, got %q", v)
}
cfg.DebugAddr = v
}
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
return Config{}, err
}
@@ -95,6 +134,30 @@ func Load(lookup Lookup) (Config, error) {
return cfg, nil
}
func LoadMigrate(lookup Lookup) (Migrate, error) {
cfg := Migrate{Env: "lab", LogLevel: slog.LevelInfo}
var err error
if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil {
return Migrate{}, err
}
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
return Migrate{}, err
}
return cfg, nil
}
func logging(lookup Lookup, env string, level slog.Level) (string, slog.Level, error) {
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
env = v
}
if v, ok := lookup("APP_LOG_LEVEL"); ok {
if err := level.UnmarshalText([]byte(v)); err != nil {
return "", 0, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v)
}
}
return env, level, nil
}
// secret reads the file named by the variable; surrounding whitespace,
// including the trailing newline editors add, is dropped.
func secret(lookup Lookup, key string) (string, error) {
+50 -8
View File
@@ -65,6 +65,9 @@ func TestDefaults(t *testing.T) {
if cfg.SessionTTL != time.Hour {
t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL)
}
if cfg.DrainDelay != 2*time.Second || cfg.IdempotencyTTL != 24*time.Hour || cfg.DBMaxConns != 10 || cfg.DebugAddr != "" {
t.Fatalf("lab3 defaults = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr)
}
}
func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) {
@@ -92,13 +95,17 @@ func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) {
func TestOverrides(t *testing.T) {
cfg, err := config.Load(env(required(t, map[string]string{
"APP_HOST": "127.0.0.1",
"APP_PORT": "9090",
"APP_ENV": "dev",
"APP_LOG_LEVEL": "debug",
"APP_SHUTDOWN_TIMEOUT": "3s",
"APP_MAX_BODY_BYTES": "1024",
"SESSION_TTL_SECONDS": "300",
"APP_HOST": "127.0.0.1",
"APP_PORT": "9090",
"APP_ENV": "dev",
"APP_LOG_LEVEL": "debug",
"APP_SHUTDOWN_TIMEOUT": "3s",
"APP_MAX_BODY_BYTES": "1024",
"SESSION_TTL_SECONDS": "300",
"APP_DRAIN_DELAY": "0s",
"IDEMPOTENCY_TTL_SECONDS": "60",
"DB_MAX_CONNS": "25",
"APP_DEBUG_ADDR": ":9090",
})))
if err != nil {
t.Fatalf("Load error: %v", err)
@@ -109,6 +116,9 @@ func TestOverrides(t *testing.T) {
if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute {
t.Fatalf("cfg = %+v", cfg)
}
if cfg.DrainDelay != 0 || cfg.IdempotencyTTL != time.Minute || cfg.DBMaxConns != 25 || cfg.DebugAddr != ":9090" {
t.Fatalf("lab3 overrides = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr)
}
}
func TestInvalidValuesAreReported(t *testing.T) {
@@ -126,10 +136,42 @@ func TestInvalidValuesAreReported(t *testing.T) {
"ttl negative": {"SESSION_TTL_SECONDS": "-5"},
"ttl text": {"SESSION_TTL_SECONDS": "hour"},
"ttl fraction": {"SESSION_TTL_SECONDS": "1.5"},
"drain negative": {"APP_DRAIN_DELAY": "-1s"},
"drain text": {"APP_DRAIN_DELAY": "abc"},
"key ttl zero": {"IDEMPOTENCY_TTL_SECONDS": "0"},
"key ttl text": {"IDEMPOTENCY_TTL_SECONDS": "day"},
"pool zero": {"DB_MAX_CONNS": "0"},
"pool too big": {"DB_MAX_CONNS": "1001"},
"pool text": {"DB_MAX_CONNS": "many"},
"debug addr": {"APP_DEBUG_ADDR": "nonsense"},
}
for name, values := range cases {
if _, err := config.Load(env(required(t, values))); err == nil {
_, err := config.Load(env(required(t, values)))
if err == nil {
t.Errorf("%s: want error", name)
continue
}
for key := range values {
if !strings.Contains(err.Error(), key) {
t.Errorf("%s: error %q does not name %s", name, err, key)
}
}
}
}
func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) {
cfg, err := config.LoadMigrate(env(map[string]string{
"DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"),
"APP_ENV": "dev",
"APP_LOG_LEVEL": "warn",
}))
if err != nil {
t.Fatalf("LoadMigrate error: %v", err)
}
if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.LogLevel != slog.LevelWarn {
t.Fatalf("cfg = %+v", cfg)
}
if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") {
t.Fatalf("missing database url: err = %v", err)
}
}
+4 -3
View File
@@ -20,8 +20,9 @@ const (
)
// NewEngine builds an engine whose every response, including routing
// failures and panics, carries X-Request-Id and the contract error body.
func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine {
// failures and panics, carries X-Instance-Id, X-Request-Id and the contract
// error body.
func NewEngine(log *slog.Logger, maxBodyBytes int64, instanceID string) *gin.Engine {
gin.SetMode(gin.ReleaseMode)
e := gin.New()
// Redirects would answer with text/plain and without a request id.
@@ -29,7 +30,7 @@ func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine {
e.RedirectFixedPath = false
e.HandleMethodNotAllowed = true
e.Use(requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes))
e.Use(instance(instanceID), requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes))
e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) })
e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") })
return e
+22 -1
View File
@@ -33,6 +33,8 @@ type payload struct {
Count int `json:"count"`
}
const testInstance = "0123456789abcdef"
func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine {
t.Helper()
return newEngineAt(t, logs, slog.LevelDebug)
@@ -44,7 +46,7 @@ func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine
logs = &bytes.Buffer{}
}
log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level}))
e := httpx.NewEngine(log, 64)
e := httpx.NewEngine(log, 64, testInstance)
e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": "<a&b>"}) })
e.GET("/panic", func(c *gin.Context) { panic("secret detail") })
e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) })
@@ -106,6 +108,25 @@ func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, sta
}
}
// The balancer spreads one client's requests over the replicas; the header
// is how a client tells which process answered, so no response may lack it.
func TestEveryResponseCarriesInstanceID(t *testing.T) {
e := newEngine(t, nil)
for _, r := range []struct{ method, path string }{
{http.MethodGet, "/ok"},
{http.MethodGet, "/missing"},
{http.MethodDelete, "/ok"},
{http.MethodGet, "/panic"},
{http.MethodGet, "/fail"},
{http.MethodPost, "/decode"},
} {
rec, _ := do(t, e, r.method, r.path, "", nil)
if got := rec.Header().Get("X-Instance-Id"); got != testInstance {
t.Errorf("%s %s (status %d): X-Instance-Id = %q, want %q", r.method, r.path, rec.Code, got, testInstance)
}
}
}
func TestUnknownRouteIsJSON404(t *testing.T) {
rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil)
assertError(t, rec, eb, http.StatusNotFound, "not_found")
+9
View File
@@ -13,6 +13,15 @@ import (
"github.com/gin-gonic/gin"
)
// instance names the process that answered, so a client behind the load
// balancer can tell the replicas apart.
func instance(id string) gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("X-Instance-Id", id)
c.Next()
}
}
// requestID assigns a fresh identifier to every request.
//
// An incoming X-Request-Id is deliberately not reused. This service is
+11
View File
@@ -7,6 +7,8 @@
package ident
import (
"crypto/rand"
"encoding/hex"
"errors"
"time"
@@ -25,6 +27,15 @@ func New() uuid.UUID {
return id
}
// NewAlias names one server process for X-Instance-Id. It is random rather
// than the hostname, so the header reveals nothing about the infrastructure,
// and a restarted process gets a new one.
func NewAlias() string {
var raw [8]byte
_, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24.
return hex.EncodeToString(raw[:])
}
// TimeOf extracts the creation time carried by a UUIDv7 as UTC with
// millisecond precision.
func TimeOf(id uuid.UUID) time.Time {
+12
View File
@@ -2,6 +2,7 @@ package ident_test
import (
"bytes"
"regexp"
"testing"
"time"
@@ -70,3 +71,14 @@ func TestParseRejectsNonCanonicalForms(t *testing.T) {
}
}
}
func TestNewAliasIs16HexAndRandom(t *testing.T) {
a, b := ident.NewAlias(), ident.NewAlias()
hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`)
if !hex16.MatchString(a) || !hex16.MatchString(b) {
t.Fatalf("aliases %q, %q are not 16 lowercase hex characters", a, b)
}
if a == b {
t.Fatalf("two aliases are equal: %q", a)
}
}
+5 -5
View File
@@ -1,8 +1,9 @@
// Package postgres opens the connection pool and applies the schema.
//
// Pool size and server-side timeouts are constants rather than settings: they
// bound how long one request can hold a connection, and the request deadline
// of the API is chosen against them.
// Server-side timeouts are constants rather than settings: they bound how
// long one request can hold a connection, and the request deadline of the API
// is chosen against them. The pool size is a setting, because the scaling
// study varies it.
package postgres
import (
@@ -27,7 +28,6 @@ import (
)
const (
maxConns = 10
connectTimeout = time.Second
statementTimeout = 5 * time.Second
idleInTxTimeout = 10 * time.Second
@@ -35,7 +35,7 @@ const (
const codeUniqueViolation = "23505"
func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, fmt.Errorf("parse database url: %w", err)
+14 -1
View File
@@ -184,8 +184,21 @@ func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) {
}
}
// The pool size is an experiment variable of the scaling study, so it is
// passed in rather than fixed.
func TestNewPoolUsesTheRequestedSize(t *testing.T) {
pool, err := postgres.NewPool(context.Background(), db.DSN, 3)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if got := pool.Config().MaxConns; got != 3 {
t.Fatalf("MaxConns = %d, want 3", got)
}
}
func TestClassifyRealConnectFailure(t *testing.T) {
_, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1")
_, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1", 1)
if err == nil {
t.Fatal("NewPool to a closed port must fail")
}
+14 -5
View File
@@ -1,6 +1,7 @@
// Package server runs the HTTP listener with the lifecycle the lab
// requires: readiness drops first, the listener stops accepting, in-flight
// requests drain within a bounded timeout, then the process exits.
// requires: readiness drops first, the process keeps serving for a drain
// delay while the load balancer notices, then the listener stops accepting,
// in-flight requests drain within a bounded timeout and the process exits.
package server
import (
@@ -17,8 +18,15 @@ type Readiness interface {
SetReady(ready bool)
}
// Run serves until ctx is cancelled, then shuts down within shutdownTimeout.
func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, shutdownTimeout time.Duration) error {
// Run serves until ctx is cancelled, keeps serving for drainDelay with
// readiness off, then shuts down within shutdownTimeout.
//
// The balancer learns about a stopping replica only from its failing
// readiness checks. Closing the listener at once would let it send requests
// to a closed port, or over a kept-alive connection the server is closing,
// until the checks catch up. drainDelay must therefore exceed the balancer's
// detection time.
func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error {
srv := &http.Server{
Handler: h,
ReadHeaderTimeout: 5 * time.Second,
@@ -45,8 +53,9 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler,
case <-ctx.Done():
}
log.Info("shutdown started", "timeout", shutdownTimeout.String())
log.Info("shutdown started", "drain_delay", drainDelay.String(), "timeout", shutdownTimeout.String())
ready.SetReady(false)
time.Sleep(drainDelay)
drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
if err := srv.Shutdown(drain); err != nil {
+46 -3
View File
@@ -19,7 +19,7 @@ import (
// newHandler builds the smallest handler with probes; the lifecycle does not
// depend on the business modules or their stores.
func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) {
e := httpx.NewEngine(log, 1024)
e := httpx.NewEngine(log, 1024, "test")
probes := health.New()
probes.Register(e)
return e, probes
@@ -34,7 +34,7 @@ func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }()
go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }()
base := "http://" + ln.Addr().String()
waitFor(t, base+"/health/ready", http.StatusOK)
@@ -62,7 +62,7 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }()
go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }()
base := "http://" + ln.Addr().String()
waitFor(t, base+"/health/ready", http.StatusOK)
@@ -85,6 +85,49 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) {
t.Fatal("readiness must be switched off during shutdown")
}
// During the drain delay the balancer is still learning that the replica is
// going away: readiness already fails, yet new connections are accepted and
// served. Only after the delay does the listener close.
func TestDrainKeepsServingWhileNotReady(t *testing.T) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
h, probes := newHandler(log)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
const drain = 700 * time.Millisecond
go func() { done <- server.Run(ctx, log, ln, h, probes, drain, 5*time.Second) }()
base := "http://" + ln.Addr().String()
waitFor(t, base+"/health/ready", http.StatusOK)
cancel()
waitFor(t, base+"/health/ready", http.StatusServiceUnavailable)
// A fresh connection, as the balancer opens for a new request.
fresh := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}}
resp, err := fresh.Get(base + "/health/live")
if err != nil {
t.Fatalf("new connection during the drain delay: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("liveness during drain = %d, want 200", resp.StatusCode)
}
select {
case err := <-done:
if err != nil {
t.Fatalf("Run returned error: %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("Run did not stop after the drain delay")
}
if _, err := fresh.Get(base + "/health/live"); err == nil {
t.Fatal("listener must be closed after the drain delay")
}
}
func waitFor(t *testing.T, url string, status int) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
+1 -1
View File
@@ -62,7 +62,7 @@ func StartPostgres(ctx context.Context) (*DB, error) {
db.Close()
return nil, fmt.Errorf("postgres address: %w", err)
}
db.Pool, err = postgres.NewPool(ctx, db.DSN)
db.Pool, err = postgres.NewPool(ctx, db.DSN, 10)
if err != nil {
db.Close()
return nil, err
+1 -1
View File
@@ -39,7 +39,7 @@ func TestMain(m *testing.M) {
func TestRunAgainstLiveServer(t *testing.T) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024})
a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"})
srv := httptest.NewServer(a.Engine)
defer srv.Close()