Ограничить ожидание зависимостей и отвечать 503 при их недоступности

This commit is contained in:
2026-10-06 15:13:04 +03:00
parent 9890482747
commit f6acdf91e0
10 changed files with 313 additions and 21 deletions
+16 -2
View File
@@ -4,6 +4,7 @@
package app
import (
"context"
"log/slog"
"time"
@@ -34,19 +35,32 @@ type Deps struct {
Sessions *redis.Client
SessionTTL time.Duration
MaxBodyBytes int64
// RequestTimeout bounds every API request; zero means the default.
RequestTimeout time.Duration
}
// defaultRequestTimeout stays below the server's 10s write timeout, so a
// request that gives up on a dependency can still write its 503.
const defaultRequestTimeout = 5 * time.Second
func New(d Deps) *App {
users := user.NewService(userstore.NewUsers(d.DB))
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes)
probes := health.New()
probes := health.New(
health.Check{Name: "postgres", Probe: d.DB.Ping},
health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }},
)
probes.Register(engine)
sessions := auth.NewService(users, authstore.NewSessions(d.Sessions), d.SessionTTL)
public := engine.Group("/api/v1")
timeout := d.RequestTimeout
if timeout == 0 {
timeout = defaultRequestTimeout
}
public := engine.Group("/api/v1", httpx.Timeout(timeout))
protected := public.Group("", auth.RequireSession(sessions))
auth.NewHandler(sessions).Register(public, protected)
user.NewHandler(users).Register(public, protected)
+100
View File
@@ -0,0 +1,100 @@
package app_test
import (
"context"
"io"
"log/slog"
"net/http"
"testing"
"time"
"messenger/internal/app"
"messenger/internal/testenv"
)
// newIsolatedTester assembles the application over containers of its own,
// so a test can stop or pause them without disturbing the shared ones.
func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *testenv.DB, *testenv.KV) {
t.Helper()
ctx := context.Background()
pg, err := testenv.StartPostgres(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pg.Close)
vk, err := testenv.StartValkey(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(vk.Close)
log := slog.New(slog.NewTextHandler(io.Discard, nil))
a := app.New(app.Deps{
Log: log,
DB: pg.Pool,
Sessions: vk.Client,
SessionTTL: time.Hour,
MaxBodyBytes: 64 * 1024,
RequestTimeout: requestTimeout,
})
return &tester{t: t, a: a}, pg, vk
}
// Without the session store no caller can be authenticated: the API answers
// 503, never 401 (the token is not at fault) and never as an anonymous user.
// Registration does not need the store and keeps working.
func TestValkeyOutageAnswers503(t *testing.T) {
ts, _, vk := newIsolatedTester(t, 5*time.Second)
alice := ts.user("alice", "Алиса")
room := ts.chat(alice.token, "t", alice.ID)
if err := vk.Stop(context.Background()); err != nil {
t.Fatal(err)
}
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+room.ID, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "")
body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword}
ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "")
}
func TestPostgresOutageAnswers503(t *testing.T) {
ts, pg, _ := newIsolatedTester(t, 5*time.Second)
alice := ts.user("alice", "Алиса")
if err := pg.Stop(context.Background()); err != nil {
t.Fatal(err)
}
ts.expect(ts.do(http.MethodGet, "/api/v1/chats", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable")
body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword}
ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusServiceUnavailable, "unavailable")
ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable")
}
// A database that accepts connections but never answers must not hold the
// request: the request deadline turns it into a 503.
func TestHungPostgresAnswers503WithinTheRequestTimeout(t *testing.T) {
ts, pg, _ := newIsolatedTester(t, 300*time.Millisecond)
alice := ts.user("alice", "Алиса")
ctx := context.Background()
if err := pg.Pause(ctx); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = pg.Unpause(ctx) })
for _, path := range []string{"/api/v1/chats", "/health/ready"} {
start := time.Now()
ts.expect(ts.do(http.MethodGet, path, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable")
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("%s took %v with a hung database, want it bounded", path, elapsed)
}
}
}
+12 -10
View File
@@ -4,7 +4,8 @@
// greater than the nil UUID, so one query serves the first page and the
// following ones, and the condition stays a range the index can scan. Ids
// are UUIDv7 and created_at derives from them, so ordering by id is the
// contract order (created_at, id).
// contract order (created_at, id). Errors pass through postgres.Classify, so
// an unreachable database surfaces as unavailability.
package store
import (
@@ -17,6 +18,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/chat"
"messenger/internal/platform/postgres"
)
type Chats struct {
@@ -34,7 +36,7 @@ const chatWithMembers = `c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id)`
func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) error {
return pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx,
`INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`,
c.ID, c.Title, c.CreatedAt); err != nil {
@@ -46,7 +48,7 @@ func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID)
return fmt.Errorf("insert chat members: %w", err)
}
return nil
})
}))
}
func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (chat.ChatWithMembers, error) {
@@ -61,7 +63,7 @@ func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (cha
return chat.ChatWithMembers{}, chat.ErrNotFound
}
if err != nil {
return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", err)
return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", postgres.Classify(err))
}
return c, nil
}
@@ -72,7 +74,7 @@ func (r *Chats) IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, e
`SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
chatID, userID).Scan(&ok)
if err != nil {
return false, fmt.Errorf("check membership: %w", err)
return false, fmt.Errorf("check membership: %w", postgres.Classify(err))
}
return ok, nil
}
@@ -88,13 +90,13 @@ func (r *Chats) ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.
LIMIT $3`,
userID, after, limit)
if err != nil {
return nil, fmt.Errorf("select chats: %w", err)
return nil, fmt.Errorf("select chats: %w", postgres.Classify(err))
}
chats, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.ChatWithMembers, error) {
return scanChat(row)
})
if err != nil {
return nil, fmt.Errorf("read chats: %w", err)
return nil, fmt.Errorf("read chats: %w", postgres.Classify(err))
}
return chats, nil
}
@@ -123,7 +125,7 @@ func (r *Messages) Append(ctx context.Context, m chat.Message) error {
`INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`,
m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt)
if err != nil {
return fmt.Errorf("insert message: %w", err)
return fmt.Errorf("insert message: %w", postgres.Classify(err))
}
return nil
}
@@ -140,7 +142,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid
LIMIT $3`,
chatID, after, limit)
if err != nil {
return nil, fmt.Errorf("select messages: %w", err)
return nil, fmt.Errorf("select messages: %w", postgres.Classify(err))
}
messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) {
var m chat.Message
@@ -149,7 +151,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid
return m, err
})
if err != nil {
return nil, fmt.Errorf("read messages: %w", err)
return nil, fmt.Errorf("read messages: %w", postgres.Classify(err))
}
return messages, nil
}
+29 -5
View File
@@ -1,11 +1,15 @@
// Package health serves the probes. Liveness only proves the process
// answers; readiness is switched off during shutdown so an orchestrator stops
// routing new traffic before the listener closes.
// answers. Readiness means the process can serve the synchronous API: it is
// switched off during shutdown, so an orchestrator stops routing new traffic
// before the listener closes, and it fails while a critical dependency is
// unreachable.
package health
import (
"context"
"net/http"
"sync/atomic"
"time"
"github.com/gin-gonic/gin"
@@ -13,16 +17,28 @@ import (
"messenger/internal/platform/httpx"
)
// checkTimeout is shared by all checks of one probe, so the probe answers
// within it even when every dependency hangs; the Docker healthcheck allows
// two seconds.
const checkTimeout = time.Second
// Check is a dependency the API cannot serve requests without.
type Check struct {
Name string
Probe func(ctx context.Context) error
}
type Handler struct {
ready atomic.Bool
ready atomic.Bool
checks []Check
}
type status struct {
Status string `json:"status"`
}
func New() *Handler {
h := &Handler{}
func New(checks ...Check) *Handler {
h := &Handler{checks: checks}
h.ready.Store(true)
return h
}
@@ -49,5 +65,13 @@ func (h *Handler) readyz(c *gin.Context) {
httpx.Error(c, apperr.Unavailable("service is shutting down"))
return
}
ctx, cancel := context.WithTimeout(c.Request.Context(), checkTimeout)
defer cancel()
for _, check := range h.checks {
if err := check.Probe(ctx); err != nil {
httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, check.Name+" is unavailable", err))
return
}
}
httpx.JSON(c, http.StatusOK, status{Status: "ok"})
}
+20
View File
@@ -307,6 +307,26 @@ func TestWithUserWithoutIdentityIsAWiringError(t *testing.T) {
assertError(t, rec, eb, http.StatusInternalServerError, "internal_error")
}
// Everything below Timeout sees one deadline, so waiting on a dependency
// ends with the request instead of holding the client.
func TestTimeoutBoundsTheRequestContext(t *testing.T) {
e := newEngine(t, nil)
e.GET("/slow", httpx.Timeout(50*time.Millisecond), func(c *gin.Context) {
select {
case <-c.Request.Context().Done():
httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, "gave up", c.Request.Context().Err()))
case <-time.After(5 * time.Second):
httpx.JSON(c, http.StatusOK, gin.H{})
}
})
start := time.Now()
rec, eb := do(t, e, http.MethodGet, "/slow", "", nil)
assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable")
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("request took %v, want it cut at the 50ms deadline", elapsed)
}
}
func logEntry(t *testing.T, logs *bytes.Buffer) map[string]any {
t.Helper()
var entry map[string]any
+13
View File
@@ -1,6 +1,7 @@
package httpx
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
@@ -93,6 +94,18 @@ func recovery() gin.HandlerFunc {
}
}
// Timeout bounds the whole request with one deadline: waiting for a pooled
// connection, SQL statements and session lookups all observe it, so a hung
// dependency ends in a 503 within d instead of holding the client.
func Timeout(d time.Duration) gin.HandlerFunc {
return func(c *gin.Context) {
ctx, cancel := context.WithTimeout(c.Request.Context(), d)
defer cancel()
c.Request = c.Request.WithContext(ctx)
c.Next()
}
}
// bodyLimit caps request bodies; exceeding it surfaces from the JSON
// decoder as *http.MaxBytesError.
func bodyLimit(maxBytes int64) gin.HandlerFunc {
+39
View File
@@ -9,6 +9,8 @@ import (
"context"
"errors"
"fmt"
"io"
"net"
"strconv"
"time"
@@ -18,6 +20,7 @@ import (
"github.com/pressly/goose/v3"
"github.com/pressly/goose/v3/lock"
"messenger/internal/platform/apperr"
"messenger/migrations"
)
@@ -71,6 +74,42 @@ func Migrate(ctx context.Context, pool *pgxpool.Pool) error {
return nil
}
// Classify marks a failure to reach the database as apperr.KindUnavailable,
// which the API answers with 503 instead of 500; any other error is returned
// unchanged. Unreachable means: the connection could not be made or broke,
// the request deadline passed, or the server refused work (PostgreSQL error
// classes 08 connection exception, 53 insufficient resources, 57 operator
// intervention such as shutdown or statement timeout).
func Classify(err error) error {
if err == nil || !unreachable(err) {
return err
}
return apperr.Wrap(apperr.KindUnavailable, "database unavailable", err)
}
func unreachable(err error) bool {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
switch class := pgErr.Code[:min(2, len(pgErr.Code))]; class {
case "08", "53", "57":
return true
}
return false
}
if _, ok := errors.AsType[*pgconn.ConnectError](err); ok {
return true
}
if _, ok := errors.AsType[net.Error](err); ok {
return true
}
return errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, context.Canceled) ||
errors.Is(err, io.EOF) ||
errors.Is(err, io.ErrUnexpectedEOF) ||
errors.Is(err, net.ErrClosed) ||
pgconn.Timeout(err) ||
pgconn.SafeToRetry(err)
}
// IsUniqueViolation reports whether err is a unique violation of the named
// constraint.
func IsUniqueViolation(err error, constraint string) bool {
@@ -4,9 +4,14 @@ import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"testing"
"github.com/jackc/pgx/v5/pgconn"
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
"messenger/internal/testenv"
)
@@ -133,3 +138,40 @@ func TestIsUniqueViolation(t *testing.T) {
t.Fatal("non-database error recognised as a violation")
}
}
func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) {
unavailable := map[string]error{
"admin shutdown": &pgconn.PgError{Code: "57P01"},
"statement timeout": &pgconn.PgError{Code: "57014"},
"connection failure": &pgconn.PgError{Code: "08006"},
"too many connections": &pgconn.PgError{Code: "53300"},
"deadline": fmt.Errorf("query: %w", context.DeadlineExceeded),
"network": &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")},
"unexpected eof": fmt.Errorf("read: %w", io.ErrUnexpectedEOF),
}
for name, err := range unavailable {
got := postgres.Classify(err)
if apperr.KindOf(got) != apperr.KindUnavailable || !errors.Is(got, err) {
t.Errorf("%s: Classify = %v (kind %v), want KindUnavailable keeping the cause", name, got, apperr.KindOf(got))
}
}
for name, err := range map[string]error{
"unique violation": &pgconn.PgError{Code: "23505"},
"plain": errors.New("boom"),
"nil": nil,
} {
if got := postgres.Classify(err); got != err {
t.Errorf("%s: Classify = %v, want the error unchanged", name, got)
}
}
}
func TestClassifyRealConnectFailure(t *testing.T) {
_, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1")
if err == nil {
t.Fatal("NewPool to a closed port must fail")
}
if apperr.KindOf(postgres.Classify(err)) != apperr.KindUnavailable {
t.Fatalf("connect failure classified as %v: %v", apperr.KindOf(postgres.Classify(err)), err)
}
}
+36
View File
@@ -10,6 +10,7 @@ import (
"runtime"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
@@ -94,6 +95,35 @@ func (d *DB) Reset(t testing.TB) {
}
}
// Stop stops the container, as when the database goes down.
func (d *DB) Stop(ctx context.Context) error {
timeout := 5 * time.Second
return d.container.Stop(ctx, &timeout)
}
// Pause freezes the server: connections stay open but nothing answers, as
// with a hung database.
func (d *DB) Pause(ctx context.Context) error {
return withDocker(ctx, func(cli *testcontainers.DockerClient) error {
return cli.ContainerPause(ctx, d.container.GetContainerID())
})
}
func (d *DB) Unpause(ctx context.Context) error {
return withDocker(ctx, func(cli *testcontainers.DockerClient) error {
return cli.ContainerUnpause(ctx, d.container.GetContainerID())
})
}
func withDocker(ctx context.Context, fn func(cli *testcontainers.DockerClient) error) error {
cli, err := testcontainers.NewDockerClientWithOpts(ctx)
if err != nil {
return err
}
defer cli.Close()
return fn(cli)
}
func (d *DB) Close() {
if d.Pool != nil {
d.Pool.Close()
@@ -136,6 +166,12 @@ func (k *KV) Reset(t testing.TB) {
}
}
// Stop stops the container, as when the session store goes down.
func (k *KV) Stop(ctx context.Context) error {
timeout := 5 * time.Second
return k.container.Stop(ctx, &timeout)
}
func (k *KV) Close() {
if k.Client != nil {
_ = k.Client.Close()
+6 -4
View File
@@ -1,4 +1,6 @@
// Package store holds the repository implementations of the users module.
// Errors pass through postgres.Classify, so an unreachable database surfaces
// as unavailability rather than as an internal error.
package store
import (
@@ -36,7 +38,7 @@ func (r *Users) Create(ctx context.Context, u user.User) error {
return user.ErrUsernameTaken
}
if err != nil {
return fmt.Errorf("insert user: %w", err)
return fmt.Errorf("insert user: %w", postgres.Classify(err))
}
return nil
}
@@ -52,11 +54,11 @@ func (r *Users) GetByUsername(ctx context.Context, username string) (user.User,
func (r *Users) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) {
rows, err := r.pool.Query(ctx, `SELECT id FROM users WHERE id = ANY($1)`, ids)
if err != nil {
return nil, fmt.Errorf("select user ids: %w", err)
return nil, fmt.Errorf("select user ids: %w", postgres.Classify(err))
}
found, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID])
if err != nil {
return nil, fmt.Errorf("read user ids: %w", err)
return nil, fmt.Errorf("read user ids: %w", postgres.Classify(err))
}
return found, nil
}
@@ -68,7 +70,7 @@ func (r *Users) one(ctx context.Context, query string, arg any) (user.User, erro
return user.User{}, user.ErrNotFound
}
if err != nil {
return user.User{}, fmt.Errorf("select user: %w", err)
return user.User{}, fmt.Errorf("select user: %w", postgres.Classify(err))
}
u.CreatedAt = u.CreatedAt.UTC()
return u, nil