Исправить замечания ревью кода: убрать повтор в AppendOnce за счёт запаса у чистки ключей, вынести MessagePayload и SameContent, принять X-Request-Id от балансировщика, упростить фоновые задачи сервера, сократить комментарии

This commit is contained in:
2026-11-04 12:19:49 +03:00
parent c9d0f73221
commit 7fbd781ca3
25 changed files with 207 additions and 247 deletions
+3 -12
View File
@@ -1,6 +1,5 @@
// Command migrate applies the embedded schema migrations once and exits. It
// runs as a one-shot service before the API replicas start, so the replicas
// never race each other over the schema and need no rights to change it.
// Command migrate applies the embedded schema migrations once and exits, so
// the API replicas never race each other over the schema.
package main
import (
@@ -24,7 +23,7 @@ func main() {
fmt.Fprintln(os.Stderr, "configuration error:", err)
os.Exit(2)
}
log := newLogger(cfg)
log := cfg.NewLogger()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
@@ -48,11 +47,3 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error {
log.Info("migrations applied", "applied_migrations", applied)
return nil
}
func newLogger(cfg config.Migrate) *slog.Logger {
opts := &slog.HandlerOptions{Level: cfg.LogLevel}
if cfg.Env == "dev" {
return slog.New(slog.NewTextHandler(os.Stdout, opts))
}
return slog.New(slog.NewJSONHandler(os.Stdout, opts))
}
+10 -24
View File
@@ -25,7 +25,6 @@ import (
"messenger/internal/server"
)
// keyCleanupEvery is how often each replica deletes expired Idempotency-Keys.
const keyCleanupEvery = 10 * time.Minute
func main() {
@@ -34,10 +33,9 @@ func main() {
fmt.Fprintln(os.Stderr, "configuration error:", err)
os.Exit(2)
}
// Every log line carries the alias that this process sends as
// X-Instance-Id, so a response can be matched to its replica's log.
alias := ident.NewAlias()
log := newLogger(cfg).With("instance", alias)
log := cfg.NewLogger().With("instance", alias)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
@@ -49,8 +47,7 @@ func main() {
}
// run connects the dependencies before it listens, so the port accepts
// connections only once they answer. The schema is applied beforehand by the
// separate migrate command, once per deployment rather than by every replica.
// connections only once they answer.
func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error {
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns)
if err != nil {
@@ -88,12 +85,8 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
publishDebugVars(alias, pool)
}
// The background work ends with the server: on a signal, and also when
// the server fails, so the deferred Wait cannot block the exit.
var background sync.WaitGroup
defer background.Wait() // before the deferred pool.Close
ctx, cancel := context.WithCancel(ctx)
defer cancel() // before Wait
var background sync.WaitGroup
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
if debugLn != nil {
background.Go(func() {
@@ -103,13 +96,13 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
})
}
return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout)
err = server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout)
cancel()
background.Wait()
return err
}
// publishDebugVars exposes, next to the runtime's memstats, what the scaling
// study needs from a replica: who it is, how many CPUs Go uses inside the
// container's limit, and how often requests had to wait for a pooled
// database connection.
// publishDebugVars adds to expvar what the load tests read from a replica.
func publishDebugVars(alias string, pool *pgxpool.Pool) {
expvar.NewString("instance").Set(alias)
expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) }))
@@ -125,14 +118,7 @@ func publishDebugVars(alias string, pool *pgxpool.Pool) {
"empty_acquire_count": s.EmptyAcquireCount(),
"empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(),
"canceled_acquire_count": s.CanceledAcquireCount(),
"new_conns_count": s.NewConnsCount(),
}
}))
}
func newLogger(cfg config.Config) *slog.Logger {
opts := &slog.HandlerOptions{Level: cfg.LogLevel}
if cfg.Env == "dev" {
return slog.New(slog.NewTextHandler(os.Stdout, opts))
}
return slog.New(slog.NewJSONHandler(os.Stdout, opts))
}
+2 -5
View File
@@ -39,10 +39,8 @@ type Deps struct {
MaxBodyBytes int64
// RequestTimeout bounds every API request; zero means the default.
RequestTimeout time.Duration
// InstanceID is sent as X-Instance-Id on every response.
InstanceID string
// IdempotencyTTL is how long a message Idempotency-Key is remembered;
// zero means the default.
InstanceID string
// IdempotencyTTL zero means the contract's 24 hours.
IdempotencyTTL time.Duration
}
@@ -50,7 +48,6 @@ type Deps struct {
// request that gives up on a dependency can still write its 503.
const defaultRequestTimeout = 5 * time.Second
// defaultIdempotencyTTL is the contract's retention of message keys.
const defaultIdempotencyTTL = 24 * time.Hour
func New(d Deps) *App {
-2
View File
@@ -52,7 +52,6 @@ func TestMain(m *testing.M) {
const testPassword = "correct-horse-battery"
// testInstance is the alias of the in-process application under test.
const testInstance = "00000000000000a1"
// errorBody mirrors the contract error envelope.
@@ -245,7 +244,6 @@ func (ts *tester) message(token, chatID, text string) chat.MessageResponse {
return decode[chat.MessageResponse](ts.t, res)
}
// freshKey is the header of a first attempt: a key never used before.
func freshKey() map[string]string {
return map[string]string{"Idempotency-Key": uuid.NewString()}
}
+2 -5
View File
@@ -146,11 +146,8 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) {
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "")
}
// The expiry is an absolute time written by the application and enforced by
// Valkey's clock. Here the test runs on the Windows host and Valkey in the
// Docker VM, whose clock lags by a fraction of a second, so the test waits
// for the 401 with a tolerance instead of sleeping exactly one TTL. On the
// stand both run in the same VM.
// Valkey enforces the expiry by the Docker VM clock, which may lag this host
// by a fraction of a second, so the 401 is awaited with a tolerance.
func TestSessionExpiresAfterTTL(t *testing.T) {
const ttl, clockTolerance = 2 * time.Second, 3 * time.Second
ts := newTesterWithTTL(t, ttl)
+2 -3
View File
@@ -101,9 +101,8 @@ func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) {
ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found")
}
// Two applications over the same stores stand for two replicas behind the
// balancer: retries of one key alternate between them and still create one
// message.
// Two applications over the same stores stand for two replicas: retries of one
// key alternate between them and still create one message.
func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
+1 -2
View File
@@ -129,8 +129,7 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) {
}
}
// ExpiresAt is written by this process's clock and enforced by Valkey's. In
// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host
// Valkey enforces the expiry by the Docker VM clock, which may lag this host
// by a fraction of a second, so the expiry is awaited with a tolerance.
func TestSessionExpiresAfterTTL(t *testing.T) {
const clockTolerance = 3 * time.Second
+1 -4
View File
@@ -6,10 +6,7 @@ import (
"time"
)
// RunKeyCleanup deletes expired Idempotency-Keys every period until ctx is
// cancelled. Every replica runs it: the deletes skip rows another replica or
// a request holds, so concurrent runs share the work instead of waiting, and
// a failed run is simply retried at the next tick.
// RunKeyCleanup purges expired Idempotency-Keys every period until ctx is done.
func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) {
ticker := time.NewTicker(every)
defer ticker.Stop()
+2 -5
View File
@@ -144,7 +144,7 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
message, err := h.chats.Send(c.Request.Context(), actor, chatID, key, req.Text)
message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text})
if err != nil {
httpx.Error(c, err)
return
@@ -173,10 +173,7 @@ func (h *Handler) history(c *gin.Context, actor uuid.UUID) {
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
// idempotencyKey reads the Idempotency-Key that names one send attempt. It
// is part of this endpoint's input contract, so it is checked here, with
// the path and the body, and before access: the answer to a malformed key
// is the same for any chat. Two headers are ambiguous and rejected.
// idempotencyKey rejects two values of the header as ambiguous.
func idempotencyKey(c *gin.Context) (string, error) {
values := c.Request.Header.Values("Idempotency-Key")
if len(values) != 1 || !keyPattern.MatchString(values[0]) {
+6
View File
@@ -42,3 +42,9 @@ func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) {
CreatedAt: ident.TimeOf(id),
}, nil
}
// SameContent reports whether two messages say the same thing, whatever their
// ids and times.
func (m Message) SameContent(other Message) bool {
return m.Text == other.Text
}
+13
View File
@@ -45,3 +45,16 @@ func TestNewMessageKeepsSurroundingWhitespace(t *testing.T) {
t.Fatalf("text must be stored verbatim, got %q", m.Text)
}
}
func TestSameContentComparesWhatTheSenderWrote(t *testing.T) {
chatID, alice := ident.New(), ident.New()
first, _ := chat.NewMessage(ident.New(), chatID, alice, "привет")
retry, _ := chat.NewMessage(ident.New(), chatID, alice, "привет")
other, _ := chat.NewMessage(ident.New(), chatID, alice, "привет!")
if !first.SameContent(retry) {
t.Fatal("messages with the same text must have the same content")
}
if first.SameContent(other) {
t.Fatal("messages with different text must differ")
}
}
+4 -6
View File
@@ -24,13 +24,11 @@ type ChatRepository interface {
}
type MessageRepository interface {
// AppendOnce stores message under the sender's key in its chat, unless
// the key was used there within ttl; it returns the message the key is
// bound to: the given one or the earlier one.
// AppendOnce stores message under the sender's key unless the key was
// used in this chat within ttl, and returns the message the key is bound
// to: the given one or the earlier one.
AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error)
// DeleteExpiredKeys removes up to limit keys older than ttl and reports
// how many it removed.
DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error)
DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error)
// ListByChat returns up to limit messages ordered by (created_at, id),
// strictly after the message identified by after when set. An unknown
// chat yields an empty history.
+24 -18
View File
@@ -12,6 +12,13 @@ import (
"messenger/internal/platform/pagination"
)
const (
keyPurgeBatch = 1000
// keyPurgeGrace keeps a key past its TTL, so a key that a request has just
// found valid cannot vanish before the request reads its message back.
keyPurgeGrace = time.Minute
)
// UserLookup is what this module needs from the users module: one batched
// existence check. It is declared here, on the consumer side, so this
// package never imports the users package and tests substitute a map.
@@ -26,8 +33,8 @@ type Service struct {
keyTTL time.Duration
}
// NewService builds the chats service; keyTTL is how long a message
// Idempotency-Key stays bound to its message.
// NewService takes keyTTL, how long an Idempotency-Key stays bound to its
// message.
func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service {
return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL}
}
@@ -66,38 +73,37 @@ func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, l
return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil
}
// Send stores a message under the sender's Idempotency-Key. Access and the
// text are checked first, on every attempt: a retry is a new request that
// must still be authorised and valid. A key already bound to a message with
// the same text returns that message; with a different text it is a conflict.
func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, key, text string) (Message, error) {
if err := s.requireMember(ctx, actor, chatID); err != nil {
// MessagePayload is one send attempt: the message and its Idempotency-Key.
type MessagePayload struct {
ChatID uuid.UUID
IdempotencyKey string
Text string
}
// Send checks access and the text on every attempt, a replay included, as the
// contract requires of a repeat.
func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) {
if err := s.requireMember(ctx, actor, p.ChatID); err != nil {
return Message{}, err
}
message, err := NewMessage(ident.New(), chatID, actor, text)
message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text)
if err != nil {
return Message{}, err
}
stored, err := s.messages.AppendOnce(ctx, message, key, s.keyTTL)
stored, err := s.messages.AppendOnce(ctx, message, p.IdempotencyKey, s.keyTTL)
if err != nil {
return Message{}, err
}
if stored.ID != message.ID && stored.Text != message.Text {
if !stored.SameContent(message) {
return Message{}, ErrKeyReused
}
return stored, nil
}
// keyPurgeBatch bounds one delete statement, so the cleanup never holds many
// row locks or runs long.
const keyPurgeBatch = 1000
// PurgeExpiredKeys deletes every key older than the retention period, batch
// by batch, and reports how many it deleted.
func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) {
var total int64
for {
n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL, keyPurgeBatch)
n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL+keyPurgeGrace, keyPurgeBatch)
total += n
if err != nil {
return total, err
+30 -9
View File
@@ -190,15 +190,15 @@ func TestSendChecksAccessBeforeValidation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = f.svc.Send(f.ctx, f.eve, c.ID, "k1", " ")
_, err = f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: " "})
if !errors.Is(err, chat.ErrNotFound) {
t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err)
}
_, err = f.svc.Send(f.ctx, f.bob, c.ID, "k2", " ")
_, err = f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: " "})
if apperr.KindOf(err) != apperr.KindInvalid {
t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err))
}
msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "k3", "Привет 👋")
msg, err := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k3", Text: "Привет 👋"})
if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" {
t.Fatalf("Send = %+v, %v", msg, err)
}
@@ -210,8 +210,8 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
if err != nil {
t.Fatal(err)
}
first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "k1", "1")
second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "k2", "2")
first, _ := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: "1"})
second, _ := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: "2"})
page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1)
if err != nil {
@@ -247,7 +247,7 @@ func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID {
func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message {
t.Helper()
m, err := f.svc.Send(f.ctx, actor, c, key, text)
m, err := f.svc.Send(f.ctx, actor, chat.MessagePayload{ChatID: c, IdempotencyKey: key, Text: text})
if err != nil {
t.Fatal(err)
}
@@ -268,7 +268,7 @@ func TestSendConflictsOnDifferentBody(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
f.send(t, f.alice, c, "k", "первое")
_, err := f.svc.Send(f.ctx, f.alice, c, "k", "другое")
_, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "другое"})
if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict {
t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err)
}
@@ -292,7 +292,7 @@ func TestSendChecksAccessBeforeKey(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
first := f.send(t, f.alice, c, "k", "секрет")
if _, err := f.svc.Send(f.ctx, f.eve, c, "k", "секрет"); !errors.Is(err, chat.ErrNotFound) {
if _, err := f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "секрет"}); !errors.Is(err, chat.ErrNotFound) {
t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err)
}
if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID {
@@ -304,7 +304,7 @@ func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
f.send(t, f.alice, c, "k", "текст")
if _, err := f.svc.Send(f.ctx, f.alice, c, "k", " "); apperr.KindOf(err) != apperr.KindInvalid {
if _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: " "}); apperr.KindOf(err) != apperr.KindInvalid {
t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err))
}
}
@@ -334,6 +334,27 @@ func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) {
}
}
// A key the claim has just seen valid must survive until the replay lookup,
// so the cleanup leaves keys alone for a while after they expire.
func TestPurgeExpiredKeysWaitsAGraceAfterTTL(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice)
f.send(t, f.alice, c, "just-expired", "1")
f.send(t, f.alice, c, "long-expired", "2")
if _, err := db.Pool.Exec(f.ctx, `
UPDATE message_idempotency_keys
SET created_at = now() - interval '1 hour' - CASE key WHEN 'just-expired' THEN interval '10 seconds' ELSE interval '10 minutes' END`); err != nil {
t.Fatal(err)
}
if deleted, err := f.svc.PurgeExpiredKeys(f.ctx); err != nil || deleted != 1 {
t.Fatalf("PurgeExpiredKeys = %d, %v; want only the long expired key", deleted, err)
}
var left string
if err := db.Pool.QueryRow(f.ctx, `SELECT key FROM message_idempotency_keys`).Scan(&left); err != nil || left != "just-expired" {
t.Fatalf("left %q, %v; want just-expired", left, err)
}
}
func TestRunKeyCleanupPurgesAndStops(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice)
-9
View File
@@ -1,9 +0,0 @@
package store
// SetAfterClaim installs a hook that runs between the claiming statement and
// the replay lookup of AppendOnce; it returns a function restoring the old one.
func SetAfterClaim(hook func()) (restore func()) {
old := afterClaim
afterClaim = hook
return func() { afterClaim = old }
}
-24
View File
@@ -9,7 +9,6 @@ import (
"github.com/google/uuid"
"messenger/internal/chat"
"messenger/internal/chat/store"
)
const keyTTL = time.Hour
@@ -168,29 +167,6 @@ func TestCleanerSkipsRowLockedByRequest(t *testing.T) {
}
}
// The key is alive when the claim runs but deleted before the replay lookup,
// as by a cleaner whose clock is a moment later at the TTL boundary. The
// operation starts over and stores the new message.
func TestAppendOnceRetriesWhenKeyVanishes(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
var once sync.Once
restore := store.SetAfterClaim(func() {
once.Do(func() {
if _, err := db.Pool.Exec(context.Background(), `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil {
t.Error(err)
}
})
})
defer restore()
next := f.newMessage(t, c.ID, alice)
if got := f.appendOnce(t, next, "k1"); got.ID != next.ID {
t.Fatalf("got %v, want the new message %v", got.ID, next.ID)
}
}
func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
+27 -52
View File
@@ -118,16 +118,9 @@ func NewMessages(pool *pgxpool.Pool) *Messages {
return &Messages{pool: pool}
}
// afterClaim runs between the claiming statement and the replay lookup of
// AppendOnce; tests use it to delete the key in that window.
var afterClaim func()
// claimSQL stores the key and the message in one statement. A key older than
// the retention period is taken over for the new message. The key row is
// inserted first: when it already exists and is still valid, claimed is
// empty and so is the message insert. One statement holds the key's lock
// only while the server executes it, never across a round trip to a replica
// that might die in between.
// claimSQL binds the key and inserts the message in one statement, so the
// key's lock never spans a round trip. An expired key is taken over; a valid
// one leaves claimed, and so the message insert, empty.
const claimSQL = `
WITH claimed AS (
INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id)
@@ -139,51 +132,33 @@ const claimSQL = `
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
SELECT $2, message_id, $1, $6, $7 FROM claimed`
// AppendOnce stores m under key unless the sender already used the key in
// this chat within ttl; it returns the message the key is bound to, m itself
// or the earlier one. Deciding whether the earlier message answers the same
// request is the caller's job.
//
// The lookup of an existing key is a second statement: within the claiming
// one, a plain SELECT would see the snapshot taken before a concurrent
// request committed the key. Between the two statements the cleaner may
// delete a key at the ttl boundary; the operation then starts over once,
// which stores m under the now free key.
// The bound message is read by a second statement: the claim's snapshot
// cannot see a key that a concurrent request has just committed.
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
for range 2 {
tag, err := r.pool.Exec(ctx, claimSQL,
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
if err != nil {
return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err))
}
if tag.RowsAffected() == 1 {
return m, nil
}
if afterClaim != nil {
afterClaim()
}
var stored chat.Message
err = r.pool.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency_keys k
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err))
}
return stored, nil
tag, err := r.pool.Exec(ctx, claimSQL,
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
if err != nil {
return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err))
}
return chat.Message{}, errors.New("idempotency key vanished twice between claim and lookup")
if tag.RowsAffected() == 1 {
return m, nil
}
var stored chat.Message
err = r.pool.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency_keys k
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
if err != nil {
return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err))
}
return stored, nil
}
// DeleteExpiredKeys removes up to limit keys older than ttl. SKIP LOCKED lets
// replicas cleaning at the same time, and requests holding a key, pass each
// other instead of waiting.
func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) {
// DeleteExpiredKeys skips rows locked by another cleaner or a request rather
// than waiting for them.
func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) {
tag, err := r.pool.Exec(ctx, `
DELETE FROM message_idempotency_keys
WHERE (user_id, chat_id, key) IN (
@@ -192,7 +167,7 @@ func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, lim
ORDER BY created_at
LIMIT $2
FOR UPDATE SKIP LOCKED)`,
interval(ttl), limit)
interval(olderThan), limit)
if err != nil {
return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err))
}
+29 -20
View File
@@ -15,31 +15,41 @@ import (
"time"
)
// Logging selects the log format, text for dev and JSON otherwise, and the
// level.
type Logging struct {
Env string
Level slog.Level
}
func (l Logging) NewLogger() *slog.Logger {
opts := &slog.HandlerOptions{Level: l.Level}
if l.Env == "dev" {
return slog.New(slog.NewTextHandler(os.Stdout, opts))
}
return slog.New(slog.NewJSONHandler(os.Stdout, opts))
}
type Config struct {
Logging
Host string
Port int
Env string
LogLevel slog.Level
ShutdownTimeout time.Duration
MaxBodyBytes int64
DatabaseURL string
SessionStoreURL string
SessionTTL time.Duration
// DrainDelay is how long a stopping process keeps serving with readiness
// already off, so the load balancer can take it out of rotation before
// the listener closes. It must exceed the balancer's detection time.
// DrainDelay keeps a stopping process serving with readiness off; it must
// exceed the balancer's detection time.
DrainDelay time.Duration
IdempotencyTTL time.Duration
DBMaxConns int32
// DebugAddr enables the internal expvar listener; empty disables it.
// DebugAddr enables the internal expvar listener when set.
DebugAddr string
}
// Migrate is the configuration of the one-shot migration command, which
// needs the database only.
type Migrate struct {
Env string
LogLevel slog.Level
Logging
DatabaseURL string
}
@@ -51,8 +61,6 @@ func Load(lookup Lookup) (Config, error) {
cfg := Config{
Host: "0.0.0.0",
Port: 8080,
Env: "lab",
LogLevel: slog.LevelInfo,
ShutdownTimeout: 8 * time.Second,
MaxBodyBytes: 64 * 1024,
SessionTTL: time.Hour,
@@ -74,7 +82,7 @@ func Load(lookup Lookup) (Config, error) {
cfg.Port = port
}
var err error
if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil {
if cfg.Logging, err = loadLogging(lookup); err != nil {
return Config{}, err
}
if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok {
@@ -135,9 +143,9 @@ func Load(lookup Lookup) (Config, error) {
}
func LoadMigrate(lookup Lookup) (Migrate, error) {
cfg := Migrate{Env: "lab", LogLevel: slog.LevelInfo}
var cfg Migrate
var err error
if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil {
if cfg.Logging, err = loadLogging(lookup); err != nil {
return Migrate{}, err
}
if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil {
@@ -146,16 +154,17 @@ func LoadMigrate(lookup Lookup) (Migrate, error) {
return cfg, nil
}
func logging(lookup Lookup, env string, level slog.Level) (string, slog.Level, error) {
func loadLogging(lookup Lookup) (Logging, error) {
l := Logging{Env: "lab", Level: slog.LevelInfo}
if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" {
env = v
l.Env = v
}
if v, ok := lookup("APP_LOG_LEVEL"); ok {
if err := level.UnmarshalText([]byte(v)); err != nil {
return "", 0, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v)
if err := l.Level.UnmarshalText([]byte(v)); err != nil {
return Logging{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v)
}
}
return env, level, nil
return l, nil
}
// secret reads the file named by the variable; surrounding whitespace,
+3 -3
View File
@@ -53,7 +53,7 @@ func TestDefaults(t *testing.T) {
if cfg.Addr() != "0.0.0.0:8080" {
t.Fatalf("Addr = %q", cfg.Addr())
}
if cfg.Env != "lab" || cfg.LogLevel != slog.LevelInfo {
if cfg.Env != "lab" || cfg.Level != slog.LevelInfo {
t.Fatalf("cfg = %+v", cfg)
}
if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 {
@@ -110,7 +110,7 @@ func TestOverrides(t *testing.T) {
if err != nil {
t.Fatalf("Load error: %v", err)
}
if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.LogLevel != slog.LevelDebug {
if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.Level != slog.LevelDebug {
t.Fatalf("cfg = %+v", cfg)
}
if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute {
@@ -168,7 +168,7 @@ func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) {
if err != nil {
t.Fatalf("LoadMigrate error: %v", err)
}
if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.LogLevel != slog.LevelWarn {
if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.Level != slog.LevelWarn {
t.Fatalf("cfg = %+v", cfg)
}
if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") {
+16 -4
View File
@@ -193,10 +193,22 @@ func TestSuccessResponseHeadersAndEscaping(t *testing.T) {
}
}
func TestRequestIDIsGeneratedNotEchoed(t *testing.T) {
rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": "client-chosen"})
if got := rec.Header().Get("X-Request-Id"); got == "" || got == "client-chosen" {
t.Fatalf("X-Request-Id = %q, want a server generated value", got)
// The balancer sets X-Request-Id on every request, overwriting what the
// client sent; the API keeps it so that one id spans both logs.
func TestRequestIDFromTheBalancerIsKept(t *testing.T) {
const fromLB = "0123456789abcdef0123456789abcdef"
rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": fromLB})
if got := rec.Header().Get("X-Request-Id"); got != fromLB {
t.Fatalf("X-Request-Id = %q, want %q", got, fromLB)
}
}
func TestMalformedRequestIDIsReplaced(t *testing.T) {
for _, bad := range []string{"has space", "semi;colon", "tab\there", strings.Repeat("a", 65)} {
rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": bad})
if got := rec.Header().Get("X-Request-Id"); got == "" || got == bad {
t.Errorf("X-Request-Id for %q = %q, want a generated value", bad, got)
}
}
}
+17 -13
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"log/slog"
"net/http"
"regexp"
"runtime/debug"
"time"
@@ -22,22 +23,17 @@ func instance(id string) gin.HandlerFunc {
}
}
// requestID assigns a fresh identifier to every request.
//
// An incoming X-Request-Id is deliberately not reused. This service is
// client-facing: there is no trusted upstream that could have minted the
// value, so honouring it would let any caller pick an arbitrary string that
// ends up in logs (log injection, unbounded length) or collides with another
// request and corrupts log correlation. Reusing the header is the right
// choice only when the caller is a trusted proxy or a sibling service that
// propagates one id across a chain of calls.
var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`)
// requestID keeps the X-Request-Id set by the load balancer, which overwrites
// any value a client sends; a malformed value is replaced, so a direct caller
// cannot inject into the logs.
func requestID(log *slog.Logger) gin.HandlerFunc {
return func(c *gin.Context) {
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
panic("httpx: random source unavailable: " + err.Error())
id := c.GetHeader("X-Request-Id")
if !requestIDPattern.MatchString(id) {
id = newRequestID()
}
id := hex.EncodeToString(raw[:])
c.Set(keyRequestID, id)
c.Set(keyLogger, log)
c.Writer.Header().Set("X-Request-Id", id)
@@ -45,6 +41,14 @@ func requestID(log *slog.Logger) gin.HandlerFunc {
}
}
func newRequestID() string {
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
panic("httpx: random source unavailable: " + err.Error())
}
return hex.EncodeToString(raw[:])
}
// accessLog writes one line per request at a level that follows the
// outcome (2xx debug, 4xx info, 5xx error), so the ordinary log level decides
// how much traffic is recorded. Headers, query strings and bodies are never
+2 -3
View File
@@ -27,9 +27,8 @@ func New() uuid.UUID {
return id
}
// NewAlias names one server process for X-Instance-Id. It is random rather
// than the hostname, so the header reveals nothing about the infrastructure,
// and a restarted process gets a new one.
// NewAlias names one server process for X-Instance-Id; random rather than the
// hostname, so the header reveals nothing about the infrastructure.
func NewAlias() string {
var raw [8]byte
_, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24.
+6 -8
View File
@@ -2,8 +2,7 @@
//
// Server-side timeouts are constants rather than settings: they bound how
// long one request can hold a connection, and the request deadline of the API
// is chosen against them. The pool size is a setting, because the scaling
// study varies it.
// is chosen against them.
package postgres
import (
@@ -75,21 +74,20 @@ func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, er
return pool, nil
}
// Migrate applies pending migrations and returns the versions it applied. A
// session advisory lock serialises concurrent starts, so several API
// replicas may call it at once; applied versions are recorded, which makes a
// repeated run a no-op.
// Migrate applies pending migrations and returns their versions; a session
// advisory lock makes overlapping runs wait for each other.
func Migrate(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) {
db := stdlib.OpenDBFromPool(pool)
defer db.Close()
locker, err := lock.NewPostgresSessionLocker()
if err != nil {
return nil, fmt.Errorf("migration lock: %w", err)
}
db := stdlib.OpenDBFromPool(pool)
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.FS, goose.WithSessionLocker(locker))
if err != nil {
db.Close()
return nil, fmt.Errorf("load migrations: %w", err)
}
defer provider.Close()
results, err := provider.Up(ctx)
if err != nil {
return nil, fmt.Errorf("apply migrations: %w", err)
@@ -190,8 +190,6 @@ func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) {
}
}
// The pool size is an experiment variable of the scaling study, so it is
// passed in rather than fixed.
func TestNewPoolUsesTheRequestedSize(t *testing.T) {
pool, err := postgres.NewPool(context.Background(), db.DSN, 3)
if err != nil {
+7 -14
View File
@@ -1,7 +1,5 @@
// Package server runs the HTTP listener with the lifecycle the lab
// requires: readiness drops first, the process keeps serving for a drain
// delay while the load balancer notices, then the listener stops accepting,
// in-flight requests drain within a bounded timeout and the process exits.
// Package server runs the HTTP listener with a graceful stop behind a load
// balancer.
package server
import (
@@ -19,13 +17,9 @@ type Readiness interface {
}
// Run serves until ctx is cancelled, keeps serving for drainDelay with
// readiness off, then shuts down within shutdownTimeout.
//
// The balancer learns about a stopping replica only from its failing
// readiness checks. Closing the listener at once would let it send requests
// to a closed port, or over a kept-alive connection the server is closing,
// until the checks catch up. drainDelay must therefore exceed the balancer's
// detection time.
// readiness off, then shuts down within shutdownTimeout. The balancer learns
// of the stop only from failing readiness checks, so drainDelay must exceed
// its detection time.
func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error {
srv := &http.Server{
Handler: h,
@@ -68,9 +62,8 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler,
return nil
}
// RunDebug serves h, the internal metrics of the process, on ln until ctx is
// cancelled. Nothing outside the stand's network reaches it and no request
// depends on it, so it closes at once instead of draining.
// RunDebug serves the internal metrics until ctx is cancelled; no request
// depends on them, so it stops without draining.
func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error {
srv := &http.Server{
Handler: h,