Сделать отправку сообщения идемпотентной по Idempotency-Key с общей таблицей ключей и их чисткой

This commit is contained in:
2026-11-02 17:11:04 +03:00
parent 7fa8881eb3
commit 951fd0d517
19 changed files with 856 additions and 40 deletions
+10
View File
@@ -9,9 +9,12 @@ import (
"os"
"os/signal"
"runtime"
"sync"
"syscall"
"time"
"messenger/internal/app"
"messenger/internal/chat"
"messenger/internal/platform/config"
"messenger/internal/platform/ident"
"messenger/internal/platform/postgres"
@@ -19,6 +22,9 @@ import (
"messenger/internal/server"
)
// keyCleanupEvery is how often each replica deletes expired Idempotency-Keys.
const keyCleanupEvery = 10 * time.Minute
func main() {
cfg, err := config.Load(os.LookupEnv)
if err != nil {
@@ -71,6 +77,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
InstanceID: alias,
IdempotencyTTL: cfg.IdempotencyTTL,
})
var background sync.WaitGroup
defer background.Wait() // before the deferred pool.Close
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout)
}
+8 -1
View File
@@ -50,9 +50,16 @@ type Deps struct {
// request that gives up on a dependency can still write its 503.
const defaultRequestTimeout = 5 * time.Second
// defaultIdempotencyTTL is the contract's retention of message keys.
const defaultIdempotencyTTL = 24 * time.Hour
func New(d Deps) *App {
users := user.NewService(userstore.NewUsers(d.DB))
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users)
keyTTL := d.IdempotencyTTL
if keyTTL == 0 {
keyTTL = defaultIdempotencyTTL
}
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL)
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
probes := health.New(
+10 -3
View File
@@ -18,6 +18,8 @@ import (
"testing"
"time"
"github.com/google/uuid"
"messenger/internal/app"
"messenger/internal/chat"
"messenger/internal/platform/pagination"
@@ -238,11 +240,16 @@ func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse
func (ts *tester) message(token, chatID, text string) chat.MessageResponse {
ts.t.Helper()
res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, nil)
res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, freshKey())
ts.expect(res, http.StatusCreated, "")
return decode[chat.MessageResponse](ts.t, res)
}
// freshKey is the header of a first attempt: a key never used before.
func freshKey() map[string]string {
return map[string]string{"Idempotency-Key": uuid.NewString()}
}
func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] {
ts.t.Helper()
res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil)
@@ -418,7 +425,7 @@ func TestMessagesEndToEnd(t *testing.T) {
t.Fatalf("page2 = %+v", page2)
}
post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, nil) }
post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, freshKey()) }
ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request")
ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request")
ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request")
@@ -480,7 +487,7 @@ func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) {
for n, c := range clientsList {
wg.Go(func() {
for i := 0; i < perClient; i++ {
res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, nil)
res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, freshKey())
if res.status != http.StatusCreated {
t.Errorf("client %d: status %d body %s", n, res.status, res.raw)
}
+20 -6
View File
@@ -146,15 +146,29 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) {
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "")
}
// The TTL leaves room for a loaded machine: with one second the first request
// after login sometimes arrived too late under -race with all packages
// running at once.
// The expiry is an absolute time written by the application and enforced by
// Valkey's clock. Here the test runs on the Windows host and Valkey in the
// Docker VM, whose clock lags by a fraction of a second, so the test waits
// for the 401 with a tolerance instead of sleeping exactly one TTL. On the
// stand both run in the same VM.
func TestSessionExpiresAfterTTL(t *testing.T) {
ts := newTesterWithTTL(t, 3*time.Second)
const ttl, clockTolerance = 2 * time.Second, 3 * time.Second
ts := newTesterWithTTL(t, ttl)
alice := ts.user("alice", "A")
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "")
time.Sleep(3200 * time.Millisecond)
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized")
time.Sleep(ttl)
deadline := time.Now().Add(clockTolerance)
for {
res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil)
if res.status == http.StatusUnauthorized {
ts.expect(res, http.StatusUnauthorized, "unauthorized")
return
}
if time.Now().After(deadline) {
t.Fatalf("session still valid %v after its TTL of %v", clockTolerance, ttl)
}
time.Sleep(100 * time.Millisecond)
}
}
func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) {
+154
View File
@@ -0,0 +1,154 @@
package app_test
import (
"bytes"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"messenger/internal/app"
)
func (ts *tester) send(token, chatID, key string, body any) response {
ts.t.Helper()
return ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, body, map[string]string{"Idempotency-Key": key})
}
func TestSendIdempotencyKeyValidation(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
room := ts.chat(alice.token, "ключи", alice.ID)
path := "/api/v1/chats/" + room.ID + "/messages"
body := map[string]string{"text": "x"}
ts.expect(ts.do(http.MethodPost, path, alice.token, body, nil), http.StatusBadRequest, "invalid_request")
for name, key := range map[string]string{
"empty": "",
"too long": strings.Repeat("a", 129),
"inner space": "a b",
"non-ascii": "ключ",
"slash": "a/b",
} {
res := ts.send(alice.token, room.ID, key, body)
if res.status != http.StatusBadRequest {
t.Errorf("%s key: status %d, want 400", name, res.status)
}
}
ts.expect(ts.send(alice.token, room.ID, strings.Repeat("a", 128), body), http.StatusCreated, "")
ts.expect(ts.send(alice.token, room.ID, "Az09._:-", body), http.StatusCreated, "")
// Two Idempotency-Key headers are ambiguous: which one names the attempt?
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"x"}`))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+alice.token)
req.Header.Add("Idempotency-Key", "one")
req.Header.Add("Idempotency-Key", "two")
rec := httptest.NewRecorder()
ts.a.Engine.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("two keys: status %d, want 400", rec.Code)
}
}
func TestReplayReturnsIdenticalBody(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
room := ts.chat(alice.token, "повторы", alice.ID)
first := ts.send(alice.token, room.ID, "retry-1", map[string]string{"text": "Привет 👋"})
ts.expect(first, http.StatusCreated, "")
// The same JSON with other spacing is the same request.
again := ts.send(alice.token, room.ID, "retry-1", ` { "text" : "Привет 👋" } `)
ts.expect(again, http.StatusCreated, "")
if !bytes.Equal(first.raw, again.raw) {
t.Fatalf("replay body differs:\n%s\n%s", first.raw, again.raw)
}
if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 {
t.Fatalf("history has %d messages, want 1", len(page.Items))
}
}
func TestReplayWithDifferentBodyIs409(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
room := ts.chat(alice.token, "конфликт", alice.ID)
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "первое"}), http.StatusCreated, "")
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "второе"}), http.StatusConflict, "conflict")
}
// A replay is a new request: a revoked session cannot fetch the stored result.
func TestReplayRechecksSession(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
room := ts.chat(alice.token, "сессия", alice.ID)
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusCreated, "")
ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusNoContent, "")
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized")
}
// The header is checked before access, like the JSON syntax: the answer does
// not depend on the chat, so it reveals nothing about a foreign one.
func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) {
ts := newTester(t)
alice, eve := ts.user("alice", "A"), ts.user("eve", "E")
room := ts.chat(alice.token, "чужой", alice.ID)
path := "/api/v1/chats/" + room.ID + "/messages"
ts.expect(ts.do(http.MethodPost, path, eve.token, map[string]string{"text": "x"}, nil), http.StatusBadRequest, "invalid_request")
ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found")
}
// Two applications over the same stores stand for two replicas behind the
// balancer: retries of one key alternate between them and still create one
// message.
func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) {
ts := newTester(t)
alice := ts.user("alice", "A")
room := ts.chat(alice.token, "реплики", alice.ID)
other := app.New(app.Deps{
Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
DB: db.Pool,
Sessions: kv.Client,
SessionTTL: time.Hour,
MaxBodyBytes: 64 * 1024,
InstanceID: "00000000000000b2",
})
engines := []http.Handler{ts.a.Engine, other.Engine}
const attempts = 10
type result struct {
status int
body string
instance string
}
results := make([]result, attempts)
var wg sync.WaitGroup
for i := range attempts {
wg.Go(func() {
req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", strings.NewReader(`{"text":"один раз"}`))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+alice.token)
req.Header.Set("Idempotency-Key", "shared")
rec := httptest.NewRecorder()
engines[i%2].ServeHTTP(rec, req)
results[i] = result{rec.Code, rec.Body.String(), rec.Header().Get("X-Instance-Id")}
})
}
wg.Wait()
instances := map[string]bool{}
for i, r := range results {
if r.status != http.StatusCreated || r.body != results[0].body {
t.Fatalf("attempt %d: %d %s; want 201 %s", i, r.status, r.body, results[0].body)
}
instances[r.instance] = true
}
if len(instances) != 2 {
t.Fatalf("instances %v, want both replicas", instances)
}
if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 {
t.Fatalf("history has %d messages, want 1: %+v", len(page.Items), page.Items)
}
}
+4
View File
@@ -28,6 +28,10 @@ const (
// member of, so outsiders cannot probe for existence.
var ErrNotFound = apperr.NotFound("chat not found")
// ErrKeyReused answers a send whose Idempotency-Key is already bound to a
// message with a different text.
var ErrKeyReused = apperr.Conflict("Idempotency-Key was already used with a different message")
type Chat struct {
ID uuid.UUID
Title string
+33
View File
@@ -0,0 +1,33 @@
package chat
import (
"context"
"log/slog"
"time"
)
// RunKeyCleanup deletes expired Idempotency-Keys every period until ctx is
// cancelled. Every replica runs it: the deletes skip rows another replica or
// a request holds, so concurrent runs share the work instead of waiting, and
// a failed run is simply retried at the next tick.
func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) {
ticker := time.NewTicker(every)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
deleted, err := svc.PurgeExpiredKeys(ctx)
if err != nil {
if ctx.Err() == nil {
log.Warn("idempotency key cleanup failed", "deleted", deleted, "error", err.Error())
}
continue
}
if deleted > 0 {
log.Info("idempotency keys purged", "deleted", deleted)
}
}
}
+21 -1
View File
@@ -2,6 +2,7 @@ package chat
import (
"net/http"
"regexp"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
@@ -133,12 +134,17 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) {
httpx.Error(c, err)
return
}
key, err := idempotencyKey(c)
if err != nil {
httpx.Error(c, err)
return
}
var req createMessageRequest
if err := httpx.DecodeJSON(c, &req); err != nil {
httpx.Error(c, err)
return
}
message, err := h.chats.Send(c.Request.Context(), actor, chatID, req.Text)
message, err := h.chats.Send(c.Request.Context(), actor, chatID, key, req.Text)
if err != nil {
httpx.Error(c, err)
return
@@ -165,6 +171,20 @@ func (h *Handler) history(c *gin.Context, actor uuid.UUID) {
httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse))
}
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
// idempotencyKey reads the Idempotency-Key that names one send attempt. It
// is part of this endpoint's input contract, so it is checked here, with
// the path and the body, and before access: the answer to a malformed key
// is the same for any chat. Two headers are ambiguous and rejected.
func idempotencyKey(c *gin.Context) (string, error) {
values := c.Request.Header.Values("Idempotency-Key")
if len(values) != 1 || !keyPattern.MatchString(values[0]) {
return "", apperr.Invalid("Idempotency-Key header is required: one value of 1-128 characters A-Z a-z 0-9 . _ : -")
}
return values[0], nil
}
func chatParam(c *gin.Context) (uuid.UUID, error) {
id, err := ident.Parse(c.Param("chat_id"))
if err != nil {
+8 -1
View File
@@ -2,6 +2,7 @@ package chat
import (
"context"
"time"
"github.com/google/uuid"
)
@@ -23,7 +24,13 @@ type ChatRepository interface {
}
type MessageRepository interface {
Append(ctx context.Context, message Message) error
// AppendOnce stores message under the sender's key in its chat, unless
// the key was used there within ttl; it returns the message the key is
// bound to: the given one or the earlier one.
AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error)
// DeleteExpiredKeys removes up to limit keys older than ttl and reports
// how many it removed.
DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error)
// ListByChat returns up to limit messages ordered by (created_at, id),
// strictly after the message identified by after when set. An unknown
// chat yields an empty history.
+37 -5
View File
@@ -3,6 +3,7 @@ package chat
import (
"context"
"fmt"
"time"
"github.com/google/uuid"
@@ -22,10 +23,13 @@ type Service struct {
chats ChatRepository
messages MessageRepository
users UserLookup
keyTTL time.Duration
}
func NewService(chats ChatRepository, messages MessageRepository, users UserLookup) *Service {
return &Service{chats: chats, messages: messages, users: users}
// NewService builds the chats service; keyTTL is how long a message
// Idempotency-Key stays bound to its message.
func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service {
return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL}
}
func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) {
@@ -62,7 +66,11 @@ func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, l
return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil
}
func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string) (Message, error) {
// Send stores a message under the sender's Idempotency-Key. Access and the
// text are checked first, on every attempt: a retry is a new request that
// must still be authorised and valid. A key already bound to a message with
// the same text returns that message; with a different text it is a conflict.
func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, key, text string) (Message, error) {
if err := s.requireMember(ctx, actor, chatID); err != nil {
return Message{}, err
}
@@ -70,10 +78,34 @@ func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string
if err != nil {
return Message{}, err
}
if err := s.messages.Append(ctx, message); err != nil {
stored, err := s.messages.AppendOnce(ctx, message, key, s.keyTTL)
if err != nil {
return Message{}, err
}
return message, nil
if stored.ID != message.ID && stored.Text != message.Text {
return Message{}, ErrKeyReused
}
return stored, nil
}
// keyPurgeBatch bounds one delete statement, so the cleanup never holds many
// row locks or runs long.
const keyPurgeBatch = 1000
// PurgeExpiredKeys deletes every key older than the retention period, batch
// by batch, and reports how many it deleted.
func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) {
var total int64
for {
n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL, keyPurgeBatch)
total += n
if err != nil {
return total, err
}
if n < keyPurgeBatch {
return total, nil
}
}
}
func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) {
+142 -6
View File
@@ -4,10 +4,13 @@ import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
@@ -67,7 +70,7 @@ func newFixture(t *testing.T) fixture {
return u.ID
}
return fixture{
svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users),
svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users, time.Hour),
users: users,
alice: register("alice"),
bob: register("bob"),
@@ -187,15 +190,15 @@ func TestSendChecksAccessBeforeValidation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = f.svc.Send(f.ctx, f.eve, c.ID, " ")
_, err = f.svc.Send(f.ctx, f.eve, c.ID, "k1", " ")
if !errors.Is(err, chat.ErrNotFound) {
t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err)
}
_, err = f.svc.Send(f.ctx, f.bob, c.ID, " ")
_, err = f.svc.Send(f.ctx, f.bob, c.ID, "k2", " ")
if apperr.KindOf(err) != apperr.KindInvalid {
t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err))
}
msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "Привет 👋")
msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "k3", "Привет 👋")
if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" {
t.Fatalf("Send = %+v, %v", msg, err)
}
@@ -207,8 +210,8 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
if err != nil {
t.Fatal(err)
}
first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "1")
second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "2")
first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "k1", "1")
second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "k2", "2")
page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1)
if err != nil {
@@ -232,3 +235,136 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err)
}
}
func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID {
t.Helper()
c, err := f.svc.Create(f.ctx, members[0], "t", members)
if err != nil {
t.Fatal(err)
}
return c.ID
}
func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message {
t.Helper()
m, err := f.svc.Send(f.ctx, actor, c, key, text)
if err != nil {
t.Fatal(err)
}
return m
}
func TestSendReplaysSameBody(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
first := f.send(t, f.alice, c, "k", "привет")
again := f.send(t, f.alice, c, "k", "привет")
if again.ID != first.ID || !again.CreatedAt.Equal(first.CreatedAt) {
t.Fatalf("replay = %+v, want %+v", again, first)
}
}
func TestSendConflictsOnDifferentBody(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
f.send(t, f.alice, c, "k", "первое")
_, err := f.svc.Send(f.ctx, f.alice, c, "k", "другое")
if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict {
t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err)
}
}
func TestKeysAreScopedByUserAndChat(t *testing.T) {
f := newFixture(t)
c1 := f.newChat(t, f.alice, f.bob)
c2 := f.newChat(t, f.alice, f.bob)
a1 := f.send(t, f.alice, c1, "k", "текст")
b1 := f.send(t, f.bob, c1, "k", "текст")
a2 := f.send(t, f.alice, c2, "k", "текст")
if a1.ID == b1.ID || a1.ID == a2.ID || b1.ID == a2.ID {
t.Fatalf("one key shared across users or chats: %v %v %v", a1.ID, b1.ID, a2.ID)
}
}
// A replay checks access again: an outsider presenting a member's key learns
// nothing and the key stays bound to the member's message.
func TestSendChecksAccessBeforeKey(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
first := f.send(t, f.alice, c, "k", "секрет")
if _, err := f.svc.Send(f.ctx, f.eve, c, "k", "секрет"); !errors.Is(err, chat.ErrNotFound) {
t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err)
}
if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID {
t.Fatalf("key rebound after the outsider: %v, want %v", again.ID, first.ID)
}
}
func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
f.send(t, f.alice, c, "k", "текст")
if _, err := f.svc.Send(f.ctx, f.alice, c, "k", " "); apperr.KindOf(err) != apperr.KindInvalid {
t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err))
}
}
func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice, f.bob)
const expired = 2500
_, err := db.Pool.Exec(f.ctx, `
WITH m AS (
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
SELECT $1, gen_random_uuid(), $2, 'old', now() FROM generate_series(1, $3)
RETURNING id)
INSERT INTO message_idempotency_keys (user_id, chat_id, key, message_id, created_at)
SELECT $2, $1, id::text, id, now() - interval '2 hours' FROM m`, c, f.alice, expired)
if err != nil {
t.Fatal(err)
}
f.send(t, f.alice, c, "fresh", "новое")
deleted, err := f.svc.PurgeExpiredKeys(f.ctx)
if err != nil || deleted != expired {
t.Fatalf("PurgeExpiredKeys = %d, %v; want %d", deleted, err, expired)
}
var left int
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 {
t.Fatalf("%d keys left, %v; want the fresh one", left, err)
}
}
func TestRunKeyCleanupPurgesAndStops(t *testing.T) {
f := newFixture(t)
c := f.newChat(t, f.alice)
f.send(t, f.alice, c, "old", "старое")
if _, err := db.Pool.Exec(f.ctx, `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(f.ctx)
stopped := make(chan struct{})
go func() {
chat.RunKeyCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), f.svc, 50*time.Millisecond)
close(stopped)
}()
deadline := time.Now().Add(2 * time.Second)
for {
var left int
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil {
t.Fatal(err)
}
if left == 0 {
break
}
if time.Now().After(deadline) {
t.Fatal("expired key was not purged within 2s")
}
time.Sleep(20 * time.Millisecond)
}
cancel()
select {
case <-stopped:
case <-time.After(time.Second):
t.Fatal("RunKeyCleanup did not return after cancel")
}
}
+9
View File
@@ -0,0 +1,9 @@
package store
// SetAfterClaim installs a hook that runs between the claiming statement and
// the replay lookup of AppendOnce; it returns a function restoring the old one.
func SetAfterClaim(hook func()) (restore func()) {
old := afterClaim
afterClaim = hook
return func() { afterClaim = old }
}
+252
View File
@@ -0,0 +1,252 @@
package store_test
import (
"context"
"sync"
"testing"
"time"
"github.com/google/uuid"
"messenger/internal/chat"
"messenger/internal/chat/store"
)
const keyTTL = time.Hour
// expire makes every stored key older than keyTTL.
func expire(t *testing.T) {
t.Helper()
if _, err := db.Pool.Exec(context.Background(),
`UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil {
t.Fatal(err)
}
}
func (f fixture) appendOnce(t *testing.T, m chat.Message, key string) chat.Message {
t.Helper()
got, err := f.messages.AppendOnce(f.ctx, m, key, keyTTL)
if err != nil {
t.Fatal(err)
}
return got
}
func (f fixture) count(t *testing.T, chatID uuid.UUID) int {
t.Helper()
got, err := f.messages.ListByChat(f.ctx, chatID, nil, 1000)
if err != nil {
t.Fatal(err)
}
return len(got)
}
func keyTarget(t *testing.T, key string) uuid.UUID {
t.Helper()
var id uuid.UUID
if err := db.Pool.QueryRow(context.Background(),
`SELECT message_id FROM message_idempotency_keys WHERE key = $1`, key).Scan(&id); err != nil {
t.Fatal(err)
}
return id
}
func TestAppendOnceInsertsThenReplays(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
first := f.newMessage(t, c.ID, alice)
if got := f.appendOnce(t, first, "k1"); got.ID != first.ID {
t.Fatalf("first append returned %v, want the new message %v", got.ID, first.ID)
}
retry := f.newMessage(t, c.ID, alice)
got := f.appendOnce(t, retry, "k1")
if got.ID != first.ID || !got.CreatedAt.Equal(first.CreatedAt) || got.Text != first.Text || got.SenderID != alice {
t.Fatalf("replay returned %+v, want the stored %+v", got, first)
}
if n := f.count(t, c.ID); n != 1 {
t.Fatalf("history has %d messages, want 1", n)
}
}
func TestAppendOnceTakesOverExpiredKey(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
expire(t)
next := f.newMessage(t, c.ID, alice)
if got := f.appendOnce(t, next, "k1"); got.ID != next.ID {
t.Fatalf("expired key returned %v, want the new message %v", got.ID, next.ID)
}
if n := f.count(t, c.ID); n != 2 {
t.Fatalf("history has %d messages, want 2", n)
}
if target := keyTarget(t, "k1"); target != next.ID {
t.Fatalf("key points to %v, want %v", target, next.ID)
}
}
// A cleaner batch holds the expired key row; the request waits for it and,
// once the row is gone, inserts its message under the now free key.
func TestAppendOnceWaitsForCleanerThenInserts(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
expire(t)
tx, err := db.Pool.Begin(f.ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(f.ctx)
if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil {
t.Fatal(err)
}
next := f.newMessage(t, c.ID, alice)
type result struct {
m chat.Message
err error
}
done := make(chan result, 1)
go func() {
m, err := f.messages.AppendOnce(f.ctx, next, "k1", keyTTL)
done <- result{m, err}
}()
select {
case r := <-done:
t.Fatalf("AppendOnce did not wait for the locked key: %+v", r)
case <-time.After(300 * time.Millisecond):
}
if _, err := tx.Exec(f.ctx, `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil {
t.Fatal(err)
}
if err := tx.Commit(f.ctx); err != nil {
t.Fatal(err)
}
select {
case r := <-done:
if r.err != nil || r.m.ID != next.ID {
t.Fatalf("after the cleaner: %+v, %v; want the new message %v", r.m, r.err, next.ID)
}
case <-time.After(5 * time.Second):
t.Fatal("AppendOnce did not finish after the cleaner committed")
}
}
// A request holding a key row must not make the cleaner wait: it skips the
// row and deletes it on a later run.
func TestCleanerSkipsRowLockedByRequest(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
expire(t)
tx, err := db.Pool.Begin(f.ctx)
if err != nil {
t.Fatal(err)
}
if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil {
t.Fatal(err)
}
start := time.Now()
n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000)
if err != nil || n != 0 {
t.Fatalf("cleaner over a locked row: deleted %d, %v; want 0", n, err)
}
if waited := time.Since(start); waited > time.Second {
t.Fatalf("cleaner waited %v for the locked row", waited)
}
if err := tx.Rollback(f.ctx); err != nil {
t.Fatal(err)
}
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000); err != nil || n != 1 {
t.Fatalf("cleaner after release: deleted %d, %v; want 1", n, err)
}
}
// The key is alive when the claim runs but deleted before the replay lookup,
// as by a cleaner whose clock is a moment later at the TTL boundary. The
// operation starts over and stores the new message.
func TestAppendOnceRetriesWhenKeyVanishes(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
var once sync.Once
restore := store.SetAfterClaim(func() {
once.Do(func() {
if _, err := db.Pool.Exec(context.Background(), `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil {
t.Error(err)
}
})
})
defer restore()
next := f.newMessage(t, c.ID, alice)
if got := f.appendOnce(t, next, "k1"); got.ID != next.ID {
t.Fatalf("got %v, want the new message %v", got.ID, next.ID)
}
}
func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
for _, k := range []string{"a", "b", "c"} {
f.appendOnce(t, f.newMessage(t, c.ID, alice), k)
}
expire(t)
f.appendOnce(t, f.newMessage(t, c.ID, alice), "fresh")
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 2 {
t.Fatalf("first batch deleted %d, %v; want 2", n, err)
}
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 1 {
t.Fatalf("second batch deleted %d, %v; want 1", n, err)
}
var left int
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 {
t.Fatalf("%d keys left, %v; want only the fresh one", left, err)
}
if n := f.count(t, c.ID); n != 4 {
t.Fatalf("cleanup removed messages: %d left, want 4", n)
}
}
// Every request brings its own new message under one key; exactly one is
// stored and all of them get it back. This also proves the key row and the
// message it references are written by one statement.
func TestConcurrentAppendOnceSameKeyStoresOneMessage(t *testing.T) {
f := newFixture(t)
alice := f.mustUser(t)
c := f.mustChat(t, alice)
const requests = 20
ids := make(chan uuid.UUID, requests)
var wg sync.WaitGroup
for range requests {
m := f.newMessage(t, c.ID, alice)
wg.Go(func() {
got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL)
if err != nil {
t.Error(err)
return
}
ids <- got.ID
})
}
wg.Wait()
close(ids)
seen := map[uuid.UUID]int{}
for id := range ids {
seen[id]++
}
if len(seen) != 1 {
t.Fatalf("requests got %d different messages, want 1: %v", len(seen), seen)
}
if n := f.count(t, c.ID); n != 1 {
t.Fatalf("history has %d messages, want 1", n)
}
}
+84 -7
View File
@@ -12,6 +12,7 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -117,14 +118,90 @@ func NewMessages(pool *pgxpool.Pool) *Messages {
return &Messages{pool: pool}
}
func (r *Messages) Append(ctx context.Context, m chat.Message) error {
_, err := r.pool.Exec(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`,
m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt)
if err != nil {
return fmt.Errorf("insert message: %w", postgres.Classify(err))
// afterClaim runs between the claiming statement and the replay lookup of
// AppendOnce; tests use it to delete the key in that window.
var afterClaim func()
// claimSQL stores the key and the message in one statement. A key older than
// the retention period is taken over for the new message. The key row is
// inserted first: when it already exists and is still valid, claimed is
// empty and so is the message insert. One statement holds the key's lock
// only while the server executes it, never across a round trip to a replica
// that might die in between.
const claimSQL = `
WITH claimed AS (
INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id)
VALUES ($1, $2, $3, $4)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET message_id = EXCLUDED.message_id, created_at = now()
WHERE k.created_at < now() - $5::interval
RETURNING message_id)
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
SELECT $2, message_id, $1, $6, $7 FROM claimed`
// AppendOnce stores m under key unless the sender already used the key in
// this chat within ttl; it returns the message the key is bound to, m itself
// or the earlier one. Deciding whether the earlier message answers the same
// request is the caller's job.
//
// The lookup of an existing key is a second statement: within the claiming
// one, a plain SELECT would see the snapshot taken before a concurrent
// request committed the key. Between the two statements the cleaner may
// delete a key at the ttl boundary; the operation then starts over once,
// which stores m under the now free key.
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
for range 2 {
tag, err := r.pool.Exec(ctx, claimSQL,
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
if err != nil {
return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err))
}
if tag.RowsAffected() == 1 {
return m, nil
}
if afterClaim != nil {
afterClaim()
}
var stored chat.Message
err = r.pool.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency_keys k
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err))
}
return stored, nil
}
return nil
return chat.Message{}, errors.New("idempotency key vanished twice between claim and lookup")
}
// DeleteExpiredKeys removes up to limit keys older than ttl. SKIP LOCKED lets
// replicas cleaning at the same time, and requests holding a key, pass each
// other instead of waiting.
func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) {
tag, err := r.pool.Exec(ctx, `
DELETE FROM message_idempotency_keys
WHERE (user_id, chat_id, key) IN (
SELECT user_id, chat_id, key FROM message_idempotency_keys
WHERE created_at < now() - $1::interval
ORDER BY created_at
LIMIT $2
FOR UPDATE SKIP LOCKED)`,
interval(ttl), limit)
if err != nil {
return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err))
}
return tag.RowsAffected(), nil
}
// interval renders d for a ::interval parameter.
func interval(d time.Duration) string {
return fmt.Sprintf("%d milliseconds", d.Milliseconds())
}
// ListByChat is the history query; scripts/explain-history.sql runs the same
+12 -5
View File
@@ -8,6 +8,7 @@ import (
"os"
"sync"
"testing"
"time"
"github.com/google/uuid"
@@ -81,6 +82,12 @@ func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message
return m
}
// appendErr stores m under a key of its own, as every send now carries one.
func (f fixture) appendErr(m chat.Message) error {
_, err := f.messages.AppendOnce(f.ctx, m, m.ID.String(), time.Hour)
return err
}
func TestChatCreateThenGetForMember(t *testing.T) {
f := newFixture(t)
alice, bob := f.mustUser(t), f.mustUser(t)
@@ -199,7 +206,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) {
c := f.mustChat(t, alice)
first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice)
for _, m := range []chat.Message{third, first, second} {
if err := f.messages.Append(f.ctx, m); err != nil {
if err := f.appendErr(m); err != nil {
t.Fatal(err)
}
}
@@ -221,13 +228,13 @@ func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) {
for range total {
m := f.newMessage(t, c.ID, alice)
expected = append(expected, m.ID)
if err := f.messages.Append(f.ctx, m); err != nil {
if err := f.appendErr(m); err != nil {
t.Fatal(err)
}
}
// Another chat's history must not leak into this one.
other := f.mustChat(t, alice)
if err := f.messages.Append(f.ctx, f.newMessage(t, other.ID, alice)); err != nil {
if err := f.appendErr(f.newMessage(t, other.ID, alice)); err != nil {
t.Fatal(err)
}
for _, limit := range []int{1, 7, 100} {
@@ -269,7 +276,7 @@ func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) {
gap := ident.New() // never stored, but sorts between a and b
b := f.newMessage(t, c.ID, alice)
for _, m := range []chat.Message{a, b} {
if err := f.messages.Append(f.ctx, m); err != nil {
if err := f.appendErr(m); err != nil {
t.Fatal(err)
}
}
@@ -291,7 +298,7 @@ func TestConcurrentAppendsKeepEveryMessage(t *testing.T) {
for range clients {
wg.Go(func() {
for range perClient {
if err := f.messages.Append(f.ctx, f.newMessage(t, c.ID, alice)); err != nil {
if err := f.appendErr(f.newMessage(t, c.ID, alice)); err != nil {
t.Error(err)
}
}
+9 -3
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"net"
"os"
"testing"
@@ -15,6 +16,7 @@ import (
"messenger/internal/platform/apperr"
"messenger/internal/platform/postgres"
"messenger/internal/testenv"
"messenger/migrations"
)
var db *testenv.DB
@@ -54,10 +56,14 @@ func TestMigrateIsRepeatable(t *testing.T) {
if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil {
t.Fatal(err)
}
if version != 1 {
t.Fatalf("schema version = %d, want 1", version)
files, err := fs.Glob(migrations.FS, "*.sql")
if err != nil {
t.Fatal(err)
}
for _, table := range []string{"users", "chats", "chat_members", "messages"} {
if version != int64(len(files)) {
t.Fatalf("schema version = %d, want %d (one per migration file)", version, len(files))
}
for _, table := range []string{"users", "chats", "chat_members", "messages", "message_idempotency_keys"} {
var exists bool
if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil {
t.Fatal(err)
@@ -0,0 +1,25 @@
-- +goose Up
-- One row per Idempotency-Key a user has sent to one chat. The contract scope
-- is user + method + chat path + key; only POST /chats/{chat_id}/messages
-- accepts keys, so (user_id, chat_id) already fixes the method and the path.
-- The primary key is what makes concurrent retries through different
-- replicas safe: the second insert of a key waits for the first one's commit.
-- The row references the message instead of storing a response: messages
-- are immutable, so a replay is answered from the source of truth.
-- The table only adds to the schema: the previous API version keeps working
-- while a rolling update runs this migration.
CREATE TABLE message_idempotency_keys (
user_id uuid NOT NULL,
chat_id uuid NOT NULL,
key text NOT NULL,
message_id uuid NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, key),
FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id)
);
-- Serves the periodic removal of keys older than the retention period.
CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at);
-- +goose Down
DROP TABLE message_idempotency_keys;
+17
View File
@@ -8,6 +8,8 @@ import (
"net/http"
"net/url"
"time"
"github.com/google/uuid"
)
// client is a minimal typed HTTP client for the messenger API.
@@ -33,6 +35,21 @@ func (c *client) post(path, token string, body, out any) error {
return c.do(req, token, http.StatusCreated, out)
}
// postMessage sends one message under a fresh Idempotency-Key.
func (c *client) postMessage(chatID, token, text string, out any) error {
raw, err := json.Marshal(map[string]string{"text": text})
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Idempotency-Key", uuid.NewString())
return c.do(req, token, http.StatusCreated, out)
}
// login opens a session with Basic credentials and returns its bearer token.
func (c *client) login(username, password string) (string, error) {
req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil)
+1 -2
View File
@@ -174,8 +174,7 @@ func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (m
<-start
for m := range perClient {
var msg chat.MessageResponse
body := map[string]string{"text": fmt.Sprintf("client %d message %d", n, m)}
err := own.post("/api/v1/chats/"+chatID+"/messages", token, body, &msg)
err := own.postMessage(chatID, token, fmt.Sprintf("client %d message %d", n, m), &msg)
mu.Lock()
if err != nil {
failed++