diff --git a/internal/app/app.go b/internal/app/app.go index 6722ee20..6653c8c5 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -4,6 +4,7 @@ package app import ( + "context" "log/slog" "time" @@ -34,19 +35,32 @@ type Deps struct { Sessions *redis.Client SessionTTL time.Duration MaxBodyBytes int64 + // RequestTimeout bounds every API request; zero means the default. + RequestTimeout time.Duration } +// defaultRequestTimeout stays below the server's 10s write timeout, so a +// request that gives up on a dependency can still write its 503. +const defaultRequestTimeout = 5 * time.Second + func New(d Deps) *App { users := user.NewService(userstore.NewUsers(d.DB)) chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users) engine := httpx.NewEngine(d.Log, d.MaxBodyBytes) - probes := health.New() + probes := health.New( + health.Check{Name: "postgres", Probe: d.DB.Ping}, + health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, + ) probes.Register(engine) sessions := auth.NewService(users, authstore.NewSessions(d.Sessions), d.SessionTTL) - public := engine.Group("/api/v1") + timeout := d.RequestTimeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + public := engine.Group("/api/v1", httpx.Timeout(timeout)) protected := public.Group("", auth.RequireSession(sessions)) auth.NewHandler(sessions).Register(public, protected) user.NewHandler(users).Register(public, protected) diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go new file mode 100644 index 00000000..cadc4ee6 --- /dev/null +++ b/internal/app/failure_test.go @@ -0,0 +1,100 @@ +package app_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +// newIsolatedTester assembles the application over containers of its own, +// so a test can stop or pause them without disturbing the shared ones. +func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *testenv.DB, *testenv.KV) { + t.Helper() + ctx := context.Background() + pg, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pg.Close) + vk, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(vk.Close) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{ + Log: log, + DB: pg.Pool, + Sessions: vk.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + RequestTimeout: requestTimeout, + }) + return &tester{t: t, a: a}, pg, vk +} + +// Without the session store no caller can be authenticated: the API answers +// 503, never 401 (the token is not at fault) and never as an anonymous user. +// Registration does not need the store and keeps working. +func TestValkeyOutageAnswers503(t *testing.T) { + ts, _, vk := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + room := ts.chat(alice.token, "t", alice.ID) + + if err := vk.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+room.ID, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") + + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") +} + +func TestPostgresOutageAnswers503(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + + if err := pg.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/chats", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") +} + +// A database that accepts connections but never answers must not hold the +// request: the request deadline turns it into a 503. +func TestHungPostgresAnswers503WithinTheRequestTimeout(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 300*time.Millisecond) + alice := ts.user("alice", "Алиса") + + ctx := context.Background() + if err := pg.Pause(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.Unpause(ctx) }) + + for _, path := range []string{"/api/v1/chats", "/health/ready"} { + start := time.Now() + ts.expect(ts.do(http.MethodGet, path, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("%s took %v with a hung database, want it bounded", path, elapsed) + } + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index 76d7b9c8..65affa06 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -4,7 +4,8 @@ // greater than the nil UUID, so one query serves the first page and the // following ones, and the condition stays a range the index can scan. Ids // are UUIDv7 and created_at derives from them, so ordering by id is the -// contract order (created_at, id). +// contract order (created_at, id). Errors pass through postgres.Classify, so +// an unreachable database surfaces as unavailability. package store import ( @@ -17,6 +18,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "messenger/internal/chat" + "messenger/internal/platform/postgres" ) type Chats struct { @@ -34,7 +36,7 @@ const chatWithMembers = `c.id, c.title, c.created_at, ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id)` func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) error { - return pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { if _, err := tx.Exec(ctx, `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`, c.ID, c.Title, c.CreatedAt); err != nil { @@ -46,7 +48,7 @@ func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) return fmt.Errorf("insert chat members: %w", err) } return nil - }) + })) } func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (chat.ChatWithMembers, error) { @@ -61,7 +63,7 @@ func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (cha return chat.ChatWithMembers{}, chat.ErrNotFound } if err != nil { - return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", err) + return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", postgres.Classify(err)) } return c, nil } @@ -72,7 +74,7 @@ func (r *Chats) IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, e `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, chatID, userID).Scan(&ok) if err != nil { - return false, fmt.Errorf("check membership: %w", err) + return false, fmt.Errorf("check membership: %w", postgres.Classify(err)) } return ok, nil } @@ -88,13 +90,13 @@ func (r *Chats) ListByMember(ctx context.Context, userID uuid.UUID, after *uuid. LIMIT $3`, userID, after, limit) if err != nil { - return nil, fmt.Errorf("select chats: %w", err) + return nil, fmt.Errorf("select chats: %w", postgres.Classify(err)) } chats, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.ChatWithMembers, error) { return scanChat(row) }) if err != nil { - return nil, fmt.Errorf("read chats: %w", err) + return nil, fmt.Errorf("read chats: %w", postgres.Classify(err)) } return chats, nil } @@ -123,7 +125,7 @@ func (r *Messages) Append(ctx context.Context, m chat.Message) error { `INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`, m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt) if err != nil { - return fmt.Errorf("insert message: %w", err) + return fmt.Errorf("insert message: %w", postgres.Classify(err)) } return nil } @@ -140,7 +142,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid LIMIT $3`, chatID, after, limit) if err != nil { - return nil, fmt.Errorf("select messages: %w", err) + return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) } messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { var m chat.Message @@ -149,7 +151,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid return m, err }) if err != nil { - return nil, fmt.Errorf("read messages: %w", err) + return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) } return messages, nil } diff --git a/internal/health/handler.go b/internal/health/handler.go index 41a7afa9..9b14e880 100644 --- a/internal/health/handler.go +++ b/internal/health/handler.go @@ -1,11 +1,15 @@ // Package health serves the probes. Liveness only proves the process -// answers; readiness is switched off during shutdown so an orchestrator stops -// routing new traffic before the listener closes. +// answers. Readiness means the process can serve the synchronous API: it is +// switched off during shutdown, so an orchestrator stops routing new traffic +// before the listener closes, and it fails while a critical dependency is +// unreachable. package health import ( + "context" "net/http" "sync/atomic" + "time" "github.com/gin-gonic/gin" @@ -13,16 +17,28 @@ import ( "messenger/internal/platform/httpx" ) +// checkTimeout is shared by all checks of one probe, so the probe answers +// within it even when every dependency hangs; the Docker healthcheck allows +// two seconds. +const checkTimeout = time.Second + +// Check is a dependency the API cannot serve requests without. +type Check struct { + Name string + Probe func(ctx context.Context) error +} + type Handler struct { - ready atomic.Bool + ready atomic.Bool + checks []Check } type status struct { Status string `json:"status"` } -func New() *Handler { - h := &Handler{} +func New(checks ...Check) *Handler { + h := &Handler{checks: checks} h.ready.Store(true) return h } @@ -49,5 +65,13 @@ func (h *Handler) readyz(c *gin.Context) { httpx.Error(c, apperr.Unavailable("service is shutting down")) return } + ctx, cancel := context.WithTimeout(c.Request.Context(), checkTimeout) + defer cancel() + for _, check := range h.checks { + if err := check.Probe(ctx); err != nil { + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, check.Name+" is unavailable", err)) + return + } + } httpx.JSON(c, http.StatusOK, status{Status: "ok"}) } diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go index f57fdedf..16ac1318 100644 --- a/internal/platform/httpx/httpx_test.go +++ b/internal/platform/httpx/httpx_test.go @@ -307,6 +307,26 @@ func TestWithUserWithoutIdentityIsAWiringError(t *testing.T) { assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") } +// Everything below Timeout sees one deadline, so waiting on a dependency +// ends with the request instead of holding the client. +func TestTimeoutBoundsTheRequestContext(t *testing.T) { + e := newEngine(t, nil) + e.GET("/slow", httpx.Timeout(50*time.Millisecond), func(c *gin.Context) { + select { + case <-c.Request.Context().Done(): + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, "gave up", c.Request.Context().Err())) + case <-time.After(5 * time.Second): + httpx.JSON(c, http.StatusOK, gin.H{}) + } + }) + start := time.Now() + rec, eb := do(t, e, http.MethodGet, "/slow", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("request took %v, want it cut at the 50ms deadline", elapsed) + } +} + func logEntry(t *testing.T, logs *bytes.Buffer) map[string]any { t.Helper() var entry map[string]any diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index 724b1830..d153a980 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -1,6 +1,7 @@ package httpx import ( + "context" "crypto/rand" "encoding/hex" "fmt" @@ -93,6 +94,18 @@ func recovery() gin.HandlerFunc { } } +// Timeout bounds the whole request with one deadline: waiting for a pooled +// connection, SQL statements and session lookups all observe it, so a hung +// dependency ends in a 503 within d instead of holding the client. +func Timeout(d time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), d) + defer cancel() + c.Request = c.Request.WithContext(ctx) + c.Next() + } +} + // bodyLimit caps request bodies; exceeding it surfaces from the JSON // decoder as *http.MaxBytesError. func bodyLimit(maxBytes int64) gin.HandlerFunc { diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go index 8ab3507b..e075fe29 100644 --- a/internal/platform/postgres/postgres.go +++ b/internal/platform/postgres/postgres.go @@ -9,6 +9,8 @@ import ( "context" "errors" "fmt" + "io" + "net" "strconv" "time" @@ -18,6 +20,7 @@ import ( "github.com/pressly/goose/v3" "github.com/pressly/goose/v3/lock" + "messenger/internal/platform/apperr" "messenger/migrations" ) @@ -71,6 +74,42 @@ func Migrate(ctx context.Context, pool *pgxpool.Pool) error { return nil } +// Classify marks a failure to reach the database as apperr.KindUnavailable, +// which the API answers with 503 instead of 500; any other error is returned +// unchanged. Unreachable means: the connection could not be made or broke, +// the request deadline passed, or the server refused work (PostgreSQL error +// classes 08 connection exception, 53 insufficient resources, 57 operator +// intervention such as shutdown or statement timeout). +func Classify(err error) error { + if err == nil || !unreachable(err) { + return err + } + return apperr.Wrap(apperr.KindUnavailable, "database unavailable", err) +} + +func unreachable(err error) bool { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + switch class := pgErr.Code[:min(2, len(pgErr.Code))]; class { + case "08", "53", "57": + return true + } + return false + } + if _, ok := errors.AsType[*pgconn.ConnectError](err); ok { + return true + } + if _, ok := errors.AsType[net.Error](err); ok { + return true + } + return errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, net.ErrClosed) || + pgconn.Timeout(err) || + pgconn.SafeToRetry(err) +} + // IsUniqueViolation reports whether err is a unique violation of the named // constraint. func IsUniqueViolation(err error, constraint string) bool { diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go index 68f67ea5..24ee4b48 100644 --- a/internal/platform/postgres/postgres_test.go +++ b/internal/platform/postgres/postgres_test.go @@ -4,9 +4,14 @@ import ( "context" "errors" "fmt" + "io" + "net" "os" "testing" + "github.com/jackc/pgx/v5/pgconn" + + "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" "messenger/internal/testenv" ) @@ -133,3 +138,40 @@ func TestIsUniqueViolation(t *testing.T) { t.Fatal("non-database error recognised as a violation") } } + +func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { + unavailable := map[string]error{ + "admin shutdown": &pgconn.PgError{Code: "57P01"}, + "statement timeout": &pgconn.PgError{Code: "57014"}, + "connection failure": &pgconn.PgError{Code: "08006"}, + "too many connections": &pgconn.PgError{Code: "53300"}, + "deadline": fmt.Errorf("query: %w", context.DeadlineExceeded), + "network": &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")}, + "unexpected eof": fmt.Errorf("read: %w", io.ErrUnexpectedEOF), + } + for name, err := range unavailable { + got := postgres.Classify(err) + if apperr.KindOf(got) != apperr.KindUnavailable || !errors.Is(got, err) { + t.Errorf("%s: Classify = %v (kind %v), want KindUnavailable keeping the cause", name, got, apperr.KindOf(got)) + } + } + for name, err := range map[string]error{ + "unique violation": &pgconn.PgError{Code: "23505"}, + "plain": errors.New("boom"), + "nil": nil, + } { + if got := postgres.Classify(err); got != err { + t.Errorf("%s: Classify = %v, want the error unchanged", name, got) + } + } +} + +func TestClassifyRealConnectFailure(t *testing.T) { + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1") + if err == nil { + t.Fatal("NewPool to a closed port must fail") + } + if apperr.KindOf(postgres.Classify(err)) != apperr.KindUnavailable { + t.Fatalf("connect failure classified as %v: %v", apperr.KindOf(postgres.Classify(err)), err) + } +} diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index 90974eda..04c9f42d 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -10,6 +10,7 @@ import ( "runtime" "sync" "testing" + "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" @@ -94,6 +95,35 @@ func (d *DB) Reset(t testing.TB) { } } +// Stop stops the container, as when the database goes down. +func (d *DB) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return d.container.Stop(ctx, &timeout) +} + +// Pause freezes the server: connections stay open but nothing answers, as +// with a hung database. +func (d *DB) Pause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerPause(ctx, d.container.GetContainerID()) + }) +} + +func (d *DB) Unpause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerUnpause(ctx, d.container.GetContainerID()) + }) +} + +func withDocker(ctx context.Context, fn func(cli *testcontainers.DockerClient) error) error { + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + return err + } + defer cli.Close() + return fn(cli) +} + func (d *DB) Close() { if d.Pool != nil { d.Pool.Close() @@ -136,6 +166,12 @@ func (k *KV) Reset(t testing.TB) { } } +// Stop stops the container, as when the session store goes down. +func (k *KV) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return k.container.Stop(ctx, &timeout) +} + func (k *KV) Close() { if k.Client != nil { _ = k.Client.Close() diff --git a/internal/user/store/postgres.go b/internal/user/store/postgres.go index 4bfda146..df5a5135 100644 --- a/internal/user/store/postgres.go +++ b/internal/user/store/postgres.go @@ -1,4 +1,6 @@ // Package store holds the repository implementations of the users module. +// Errors pass through postgres.Classify, so an unreachable database surfaces +// as unavailability rather than as an internal error. package store import ( @@ -36,7 +38,7 @@ func (r *Users) Create(ctx context.Context, u user.User) error { return user.ErrUsernameTaken } if err != nil { - return fmt.Errorf("insert user: %w", err) + return fmt.Errorf("insert user: %w", postgres.Classify(err)) } return nil } @@ -52,11 +54,11 @@ func (r *Users) GetByUsername(ctx context.Context, username string) (user.User, func (r *Users) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { rows, err := r.pool.Query(ctx, `SELECT id FROM users WHERE id = ANY($1)`, ids) if err != nil { - return nil, fmt.Errorf("select user ids: %w", err) + return nil, fmt.Errorf("select user ids: %w", postgres.Classify(err)) } found, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) if err != nil { - return nil, fmt.Errorf("read user ids: %w", err) + return nil, fmt.Errorf("read user ids: %w", postgres.Classify(err)) } return found, nil } @@ -68,7 +70,7 @@ func (r *Users) one(ctx context.Context, query string, arg any) (user.User, erro return user.User{}, user.ErrNotFound } if err != nil { - return user.User{}, fmt.Errorf("select user: %w", err) + return user.User{}, fmt.Errorf("select user: %w", postgres.Classify(err)) } u.CreatedAt = u.CreatedAt.UTC() return u, nil