Сделать отправку сообщения идемпотентной по Idempotency-Key с общей таблицей ключей и их чисткой
This commit is contained in:
@@ -9,9 +9,12 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/platform/postgres"
|
||||
@@ -19,6 +22,9 @@ import (
|
||||
"messenger/internal/server"
|
||||
)
|
||||
|
||||
// keyCleanupEvery is how often each replica deletes expired Idempotency-Keys.
|
||||
const keyCleanupEvery = 10 * time.Minute
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load(os.LookupEnv)
|
||||
if err != nil {
|
||||
@@ -71,6 +77,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
InstanceID: alias,
|
||||
IdempotencyTTL: cfg.IdempotencyTTL,
|
||||
})
|
||||
var background sync.WaitGroup
|
||||
defer background.Wait() // before the deferred pool.Close
|
||||
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
|
||||
|
||||
return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout)
|
||||
}
|
||||
|
||||
|
||||
+8
-1
@@ -50,9 +50,16 @@ type Deps struct {
|
||||
// request that gives up on a dependency can still write its 503.
|
||||
const defaultRequestTimeout = 5 * time.Second
|
||||
|
||||
// defaultIdempotencyTTL is the contract's retention of message keys.
|
||||
const defaultIdempotencyTTL = 24 * time.Hour
|
||||
|
||||
func New(d Deps) *App {
|
||||
users := user.NewService(userstore.NewUsers(d.DB))
|
||||
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users)
|
||||
keyTTL := d.IdempotencyTTL
|
||||
if keyTTL == 0 {
|
||||
keyTTL = defaultIdempotencyTTL
|
||||
}
|
||||
chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL)
|
||||
|
||||
engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID)
|
||||
probes := health.New(
|
||||
|
||||
@@ -18,6 +18,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/platform/pagination"
|
||||
@@ -238,11 +240,16 @@ func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse
|
||||
|
||||
func (ts *tester) message(token, chatID, text string) chat.MessageResponse {
|
||||
ts.t.Helper()
|
||||
res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, nil)
|
||||
res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, freshKey())
|
||||
ts.expect(res, http.StatusCreated, "")
|
||||
return decode[chat.MessageResponse](ts.t, res)
|
||||
}
|
||||
|
||||
// freshKey is the header of a first attempt: a key never used before.
|
||||
func freshKey() map[string]string {
|
||||
return map[string]string{"Idempotency-Key": uuid.NewString()}
|
||||
}
|
||||
|
||||
func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] {
|
||||
ts.t.Helper()
|
||||
res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil)
|
||||
@@ -418,7 +425,7 @@ func TestMessagesEndToEnd(t *testing.T) {
|
||||
t.Fatalf("page2 = %+v", page2)
|
||||
}
|
||||
|
||||
post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, nil) }
|
||||
post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, freshKey()) }
|
||||
ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request")
|
||||
ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request")
|
||||
ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request")
|
||||
@@ -480,7 +487,7 @@ func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) {
|
||||
for n, c := range clientsList {
|
||||
wg.Go(func() {
|
||||
for i := 0; i < perClient; i++ {
|
||||
res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, nil)
|
||||
res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, freshKey())
|
||||
if res.status != http.StatusCreated {
|
||||
t.Errorf("client %d: status %d body %s", n, res.status, res.raw)
|
||||
}
|
||||
|
||||
@@ -146,15 +146,29 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) {
|
||||
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "")
|
||||
}
|
||||
|
||||
// The TTL leaves room for a loaded machine: with one second the first request
|
||||
// after login sometimes arrived too late under -race with all packages
|
||||
// running at once.
|
||||
// The expiry is an absolute time written by the application and enforced by
|
||||
// Valkey's clock. Here the test runs on the Windows host and Valkey in the
|
||||
// Docker VM, whose clock lags by a fraction of a second, so the test waits
|
||||
// for the 401 with a tolerance instead of sleeping exactly one TTL. On the
|
||||
// stand both run in the same VM.
|
||||
func TestSessionExpiresAfterTTL(t *testing.T) {
|
||||
ts := newTesterWithTTL(t, 3*time.Second)
|
||||
const ttl, clockTolerance = 2 * time.Second, 3 * time.Second
|
||||
ts := newTesterWithTTL(t, ttl)
|
||||
alice := ts.user("alice", "A")
|
||||
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "")
|
||||
time.Sleep(3200 * time.Millisecond)
|
||||
ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized")
|
||||
time.Sleep(ttl)
|
||||
deadline := time.Now().Add(clockTolerance)
|
||||
for {
|
||||
res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil)
|
||||
if res.status == http.StatusUnauthorized {
|
||||
ts.expect(res, http.StatusUnauthorized, "unauthorized")
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("session still valid %v after its TTL of %v", clockTolerance, ttl)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package app_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/app"
|
||||
)
|
||||
|
||||
func (ts *tester) send(token, chatID, key string, body any) response {
|
||||
ts.t.Helper()
|
||||
return ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, body, map[string]string{"Idempotency-Key": key})
|
||||
}
|
||||
|
||||
func TestSendIdempotencyKeyValidation(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice := ts.user("alice", "A")
|
||||
room := ts.chat(alice.token, "ключи", alice.ID)
|
||||
path := "/api/v1/chats/" + room.ID + "/messages"
|
||||
body := map[string]string{"text": "x"}
|
||||
|
||||
ts.expect(ts.do(http.MethodPost, path, alice.token, body, nil), http.StatusBadRequest, "invalid_request")
|
||||
for name, key := range map[string]string{
|
||||
"empty": "",
|
||||
"too long": strings.Repeat("a", 129),
|
||||
"inner space": "a b",
|
||||
"non-ascii": "ключ",
|
||||
"slash": "a/b",
|
||||
} {
|
||||
res := ts.send(alice.token, room.ID, key, body)
|
||||
if res.status != http.StatusBadRequest {
|
||||
t.Errorf("%s key: status %d, want 400", name, res.status)
|
||||
}
|
||||
}
|
||||
ts.expect(ts.send(alice.token, room.ID, strings.Repeat("a", 128), body), http.StatusCreated, "")
|
||||
ts.expect(ts.send(alice.token, room.ID, "Az09._:-", body), http.StatusCreated, "")
|
||||
|
||||
// Two Idempotency-Key headers are ambiguous: which one names the attempt?
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"x"}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+alice.token)
|
||||
req.Header.Add("Idempotency-Key", "one")
|
||||
req.Header.Add("Idempotency-Key", "two")
|
||||
rec := httptest.NewRecorder()
|
||||
ts.a.Engine.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("two keys: status %d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayReturnsIdenticalBody(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice := ts.user("alice", "A")
|
||||
room := ts.chat(alice.token, "повторы", alice.ID)
|
||||
first := ts.send(alice.token, room.ID, "retry-1", map[string]string{"text": "Привет 👋"})
|
||||
ts.expect(first, http.StatusCreated, "")
|
||||
// The same JSON with other spacing is the same request.
|
||||
again := ts.send(alice.token, room.ID, "retry-1", ` { "text" : "Привет 👋" } `)
|
||||
ts.expect(again, http.StatusCreated, "")
|
||||
if !bytes.Equal(first.raw, again.raw) {
|
||||
t.Fatalf("replay body differs:\n%s\n%s", first.raw, again.raw)
|
||||
}
|
||||
if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 {
|
||||
t.Fatalf("history has %d messages, want 1", len(page.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayWithDifferentBodyIs409(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice := ts.user("alice", "A")
|
||||
room := ts.chat(alice.token, "конфликт", alice.ID)
|
||||
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "первое"}), http.StatusCreated, "")
|
||||
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "второе"}), http.StatusConflict, "conflict")
|
||||
}
|
||||
|
||||
// A replay is a new request: a revoked session cannot fetch the stored result.
|
||||
func TestReplayRechecksSession(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice := ts.user("alice", "A")
|
||||
room := ts.chat(alice.token, "сессия", alice.ID)
|
||||
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusCreated, "")
|
||||
ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusNoContent, "")
|
||||
ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
|
||||
// The header is checked before access, like the JSON syntax: the answer does
|
||||
// not depend on the chat, so it reveals nothing about a foreign one.
|
||||
func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice, eve := ts.user("alice", "A"), ts.user("eve", "E")
|
||||
room := ts.chat(alice.token, "чужой", alice.ID)
|
||||
path := "/api/v1/chats/" + room.ID + "/messages"
|
||||
ts.expect(ts.do(http.MethodPost, path, eve.token, map[string]string{"text": "x"}, nil), http.StatusBadRequest, "invalid_request")
|
||||
ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found")
|
||||
}
|
||||
|
||||
// Two applications over the same stores stand for two replicas behind the
|
||||
// balancer: retries of one key alternate between them and still create one
|
||||
// message.
|
||||
func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) {
|
||||
ts := newTester(t)
|
||||
alice := ts.user("alice", "A")
|
||||
room := ts.chat(alice.token, "реплики", alice.ID)
|
||||
other := app.New(app.Deps{
|
||||
Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
DB: db.Pool,
|
||||
Sessions: kv.Client,
|
||||
SessionTTL: time.Hour,
|
||||
MaxBodyBytes: 64 * 1024,
|
||||
InstanceID: "00000000000000b2",
|
||||
})
|
||||
engines := []http.Handler{ts.a.Engine, other.Engine}
|
||||
|
||||
const attempts = 10
|
||||
type result struct {
|
||||
status int
|
||||
body string
|
||||
instance string
|
||||
}
|
||||
results := make([]result, attempts)
|
||||
var wg sync.WaitGroup
|
||||
for i := range attempts {
|
||||
wg.Go(func() {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", strings.NewReader(`{"text":"один раз"}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+alice.token)
|
||||
req.Header.Set("Idempotency-Key", "shared")
|
||||
rec := httptest.NewRecorder()
|
||||
engines[i%2].ServeHTTP(rec, req)
|
||||
results[i] = result{rec.Code, rec.Body.String(), rec.Header().Get("X-Instance-Id")}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
instances := map[string]bool{}
|
||||
for i, r := range results {
|
||||
if r.status != http.StatusCreated || r.body != results[0].body {
|
||||
t.Fatalf("attempt %d: %d %s; want 201 %s", i, r.status, r.body, results[0].body)
|
||||
}
|
||||
instances[r.instance] = true
|
||||
}
|
||||
if len(instances) != 2 {
|
||||
t.Fatalf("instances %v, want both replicas", instances)
|
||||
}
|
||||
if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 {
|
||||
t.Fatalf("history has %d messages, want 1: %+v", len(page.Items), page.Items)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,10 @@ const (
|
||||
// member of, so outsiders cannot probe for existence.
|
||||
var ErrNotFound = apperr.NotFound("chat not found")
|
||||
|
||||
// ErrKeyReused answers a send whose Idempotency-Key is already bound to a
|
||||
// message with a different text.
|
||||
var ErrKeyReused = apperr.Conflict("Idempotency-Key was already used with a different message")
|
||||
|
||||
type Chat struct {
|
||||
ID uuid.UUID
|
||||
Title string
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RunKeyCleanup deletes expired Idempotency-Keys every period until ctx is
|
||||
// cancelled. Every replica runs it: the deletes skip rows another replica or
|
||||
// a request holds, so concurrent runs share the work instead of waiting, and
|
||||
// a failed run is simply retried at the next tick.
|
||||
func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
deleted, err := svc.PurgeExpiredKeys(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Warn("idempotency key cleanup failed", "deleted", deleted, "error", err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Info("idempotency keys purged", "deleted", deleted)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package chat
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
@@ -133,12 +134,17 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
key, err := idempotencyKey(c)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
var req createMessageRequest
|
||||
if err := httpx.DecodeJSON(c, &req); err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
}
|
||||
message, err := h.chats.Send(c.Request.Context(), actor, chatID, req.Text)
|
||||
message, err := h.chats.Send(c.Request.Context(), actor, chatID, key, req.Text)
|
||||
if err != nil {
|
||||
httpx.Error(c, err)
|
||||
return
|
||||
@@ -165,6 +171,20 @@ func (h *Handler) history(c *gin.Context, actor uuid.UUID) {
|
||||
httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse))
|
||||
}
|
||||
|
||||
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
// idempotencyKey reads the Idempotency-Key that names one send attempt. It
|
||||
// is part of this endpoint's input contract, so it is checked here, with
|
||||
// the path and the body, and before access: the answer to a malformed key
|
||||
// is the same for any chat. Two headers are ambiguous and rejected.
|
||||
func idempotencyKey(c *gin.Context) (string, error) {
|
||||
values := c.Request.Header.Values("Idempotency-Key")
|
||||
if len(values) != 1 || !keyPattern.MatchString(values[0]) {
|
||||
return "", apperr.Invalid("Idempotency-Key header is required: one value of 1-128 characters A-Z a-z 0-9 . _ : -")
|
||||
}
|
||||
return values[0], nil
|
||||
}
|
||||
|
||||
func chatParam(c *gin.Context) (uuid.UUID, error) {
|
||||
id, err := ident.Parse(c.Param("chat_id"))
|
||||
if err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -23,7 +24,13 @@ type ChatRepository interface {
|
||||
}
|
||||
|
||||
type MessageRepository interface {
|
||||
Append(ctx context.Context, message Message) error
|
||||
// AppendOnce stores message under the sender's key in its chat, unless
|
||||
// the key was used there within ttl; it returns the message the key is
|
||||
// bound to: the given one or the earlier one.
|
||||
AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error)
|
||||
// DeleteExpiredKeys removes up to limit keys older than ttl and reports
|
||||
// how many it removed.
|
||||
DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error)
|
||||
// ListByChat returns up to limit messages ordered by (created_at, id),
|
||||
// strictly after the message identified by after when set. An unknown
|
||||
// chat yields an empty history.
|
||||
|
||||
@@ -3,6 +3,7 @@ package chat
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
@@ -22,10 +23,13 @@ type Service struct {
|
||||
chats ChatRepository
|
||||
messages MessageRepository
|
||||
users UserLookup
|
||||
keyTTL time.Duration
|
||||
}
|
||||
|
||||
func NewService(chats ChatRepository, messages MessageRepository, users UserLookup) *Service {
|
||||
return &Service{chats: chats, messages: messages, users: users}
|
||||
// NewService builds the chats service; keyTTL is how long a message
|
||||
// Idempotency-Key stays bound to its message.
|
||||
func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service {
|
||||
return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL}
|
||||
}
|
||||
|
||||
func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) {
|
||||
@@ -62,7 +66,11 @@ func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, l
|
||||
return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil
|
||||
}
|
||||
|
||||
func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string) (Message, error) {
|
||||
// Send stores a message under the sender's Idempotency-Key. Access and the
|
||||
// text are checked first, on every attempt: a retry is a new request that
|
||||
// must still be authorised and valid. A key already bound to a message with
|
||||
// the same text returns that message; with a different text it is a conflict.
|
||||
func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, key, text string) (Message, error) {
|
||||
if err := s.requireMember(ctx, actor, chatID); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
@@ -70,10 +78,34 @@ func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
if err := s.messages.Append(ctx, message); err != nil {
|
||||
stored, err := s.messages.AppendOnce(ctx, message, key, s.keyTTL)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return message, nil
|
||||
if stored.ID != message.ID && stored.Text != message.Text {
|
||||
return Message{}, ErrKeyReused
|
||||
}
|
||||
return stored, nil
|
||||
}
|
||||
|
||||
// keyPurgeBatch bounds one delete statement, so the cleanup never holds many
|
||||
// row locks or runs long.
|
||||
const keyPurgeBatch = 1000
|
||||
|
||||
// PurgeExpiredKeys deletes every key older than the retention period, batch
|
||||
// by batch, and reports how many it deleted.
|
||||
func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) {
|
||||
var total int64
|
||||
for {
|
||||
n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL, keyPurgeBatch)
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
if n < keyPurgeBatch {
|
||||
return total, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) {
|
||||
|
||||
@@ -4,10 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
@@ -67,7 +70,7 @@ func newFixture(t *testing.T) fixture {
|
||||
return u.ID
|
||||
}
|
||||
return fixture{
|
||||
svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users),
|
||||
svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users, time.Hour),
|
||||
users: users,
|
||||
alice: register("alice"),
|
||||
bob: register("bob"),
|
||||
@@ -187,15 +190,15 @@ func TestSendChecksAccessBeforeValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = f.svc.Send(f.ctx, f.eve, c.ID, " ")
|
||||
_, err = f.svc.Send(f.ctx, f.eve, c.ID, "k1", " ")
|
||||
if !errors.Is(err, chat.ErrNotFound) {
|
||||
t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err)
|
||||
}
|
||||
_, err = f.svc.Send(f.ctx, f.bob, c.ID, " ")
|
||||
_, err = f.svc.Send(f.ctx, f.bob, c.ID, "k2", " ")
|
||||
if apperr.KindOf(err) != apperr.KindInvalid {
|
||||
t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err))
|
||||
}
|
||||
msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "Привет 👋")
|
||||
msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "k3", "Привет 👋")
|
||||
if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" {
|
||||
t.Fatalf("Send = %+v, %v", msg, err)
|
||||
}
|
||||
@@ -207,8 +210,8 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "1")
|
||||
second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "2")
|
||||
first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "k1", "1")
|
||||
second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "k2", "2")
|
||||
|
||||
page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1)
|
||||
if err != nil {
|
||||
@@ -232,3 +235,136 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) {
|
||||
t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID {
|
||||
t.Helper()
|
||||
c, err := f.svc.Create(f.ctx, members[0], "t", members)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c.ID
|
||||
}
|
||||
|
||||
func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message {
|
||||
t.Helper()
|
||||
m, err := f.svc.Send(f.ctx, actor, c, key, text)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestSendReplaysSameBody(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
first := f.send(t, f.alice, c, "k", "привет")
|
||||
again := f.send(t, f.alice, c, "k", "привет")
|
||||
if again.ID != first.ID || !again.CreatedAt.Equal(first.CreatedAt) {
|
||||
t.Fatalf("replay = %+v, want %+v", again, first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendConflictsOnDifferentBody(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
f.send(t, f.alice, c, "k", "первое")
|
||||
_, err := f.svc.Send(f.ctx, f.alice, c, "k", "другое")
|
||||
if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict {
|
||||
t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeysAreScopedByUserAndChat(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c1 := f.newChat(t, f.alice, f.bob)
|
||||
c2 := f.newChat(t, f.alice, f.bob)
|
||||
a1 := f.send(t, f.alice, c1, "k", "текст")
|
||||
b1 := f.send(t, f.bob, c1, "k", "текст")
|
||||
a2 := f.send(t, f.alice, c2, "k", "текст")
|
||||
if a1.ID == b1.ID || a1.ID == a2.ID || b1.ID == a2.ID {
|
||||
t.Fatalf("one key shared across users or chats: %v %v %v", a1.ID, b1.ID, a2.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// A replay checks access again: an outsider presenting a member's key learns
|
||||
// nothing and the key stays bound to the member's message.
|
||||
func TestSendChecksAccessBeforeKey(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
first := f.send(t, f.alice, c, "k", "секрет")
|
||||
if _, err := f.svc.Send(f.ctx, f.eve, c, "k", "секрет"); !errors.Is(err, chat.ErrNotFound) {
|
||||
t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err)
|
||||
}
|
||||
if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID {
|
||||
t.Fatalf("key rebound after the outsider: %v, want %v", again.ID, first.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
f.send(t, f.alice, c, "k", "текст")
|
||||
if _, err := f.svc.Send(f.ctx, f.alice, c, "k", " "); apperr.KindOf(err) != apperr.KindInvalid {
|
||||
t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice, f.bob)
|
||||
const expired = 2500
|
||||
_, err := db.Pool.Exec(f.ctx, `
|
||||
WITH m AS (
|
||||
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
|
||||
SELECT $1, gen_random_uuid(), $2, 'old', now() FROM generate_series(1, $3)
|
||||
RETURNING id)
|
||||
INSERT INTO message_idempotency_keys (user_id, chat_id, key, message_id, created_at)
|
||||
SELECT $2, $1, id::text, id, now() - interval '2 hours' FROM m`, c, f.alice, expired)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.send(t, f.alice, c, "fresh", "новое")
|
||||
deleted, err := f.svc.PurgeExpiredKeys(f.ctx)
|
||||
if err != nil || deleted != expired {
|
||||
t.Fatalf("PurgeExpiredKeys = %d, %v; want %d", deleted, err, expired)
|
||||
}
|
||||
var left int
|
||||
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 {
|
||||
t.Fatalf("%d keys left, %v; want the fresh one", left, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunKeyCleanupPurgesAndStops(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
c := f.newChat(t, f.alice)
|
||||
f.send(t, f.alice, c, "old", "старое")
|
||||
if _, err := db.Pool.Exec(f.ctx, `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
chat.RunKeyCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), f.svc, 50*time.Millisecond)
|
||||
close(stopped)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
var left int
|
||||
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("expired key was not purged within 2s")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("RunKeyCleanup did not return after cancel")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package store
|
||||
|
||||
// SetAfterClaim installs a hook that runs between the claiming statement and
|
||||
// the replay lookup of AppendOnce; it returns a function restoring the old one.
|
||||
func SetAfterClaim(hook func()) (restore func()) {
|
||||
old := afterClaim
|
||||
afterClaim = hook
|
||||
return func() { afterClaim = old }
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package store_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/chat/store"
|
||||
)
|
||||
|
||||
const keyTTL = time.Hour
|
||||
|
||||
// expire makes every stored key older than keyTTL.
|
||||
func expire(t *testing.T) {
|
||||
t.Helper()
|
||||
if _, err := db.Pool.Exec(context.Background(),
|
||||
`UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f fixture) appendOnce(t *testing.T, m chat.Message, key string) chat.Message {
|
||||
t.Helper()
|
||||
got, err := f.messages.AppendOnce(f.ctx, m, key, keyTTL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func (f fixture) count(t *testing.T, chatID uuid.UUID) int {
|
||||
t.Helper()
|
||||
got, err := f.messages.ListByChat(f.ctx, chatID, nil, 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(got)
|
||||
}
|
||||
|
||||
func keyTarget(t *testing.T, key string) uuid.UUID {
|
||||
t.Helper()
|
||||
var id uuid.UUID
|
||||
if err := db.Pool.QueryRow(context.Background(),
|
||||
`SELECT message_id FROM message_idempotency_keys WHERE key = $1`, key).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestAppendOnceInsertsThenReplays(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
first := f.newMessage(t, c.ID, alice)
|
||||
if got := f.appendOnce(t, first, "k1"); got.ID != first.ID {
|
||||
t.Fatalf("first append returned %v, want the new message %v", got.ID, first.ID)
|
||||
}
|
||||
retry := f.newMessage(t, c.ID, alice)
|
||||
got := f.appendOnce(t, retry, "k1")
|
||||
if got.ID != first.ID || !got.CreatedAt.Equal(first.CreatedAt) || got.Text != first.Text || got.SenderID != alice {
|
||||
t.Fatalf("replay returned %+v, want the stored %+v", got, first)
|
||||
}
|
||||
if n := f.count(t, c.ID); n != 1 {
|
||||
t.Fatalf("history has %d messages, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendOnceTakesOverExpiredKey(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
|
||||
expire(t)
|
||||
next := f.newMessage(t, c.ID, alice)
|
||||
if got := f.appendOnce(t, next, "k1"); got.ID != next.ID {
|
||||
t.Fatalf("expired key returned %v, want the new message %v", got.ID, next.ID)
|
||||
}
|
||||
if n := f.count(t, c.ID); n != 2 {
|
||||
t.Fatalf("history has %d messages, want 2", n)
|
||||
}
|
||||
if target := keyTarget(t, "k1"); target != next.ID {
|
||||
t.Fatalf("key points to %v, want %v", target, next.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// A cleaner batch holds the expired key row; the request waits for it and,
|
||||
// once the row is gone, inserts its message under the now free key.
|
||||
func TestAppendOnceWaitsForCleanerThenInserts(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
|
||||
expire(t)
|
||||
|
||||
tx, err := db.Pool.Begin(f.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(f.ctx)
|
||||
if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
next := f.newMessage(t, c.ID, alice)
|
||||
type result struct {
|
||||
m chat.Message
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
m, err := f.messages.AppendOnce(f.ctx, next, "k1", keyTTL)
|
||||
done <- result{m, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
t.Fatalf("AppendOnce did not wait for the locked key: %+v", r)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
if _, err := tx.Exec(f.ctx, `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case r := <-done:
|
||||
if r.err != nil || r.m.ID != next.ID {
|
||||
t.Fatalf("after the cleaner: %+v, %v; want the new message %v", r.m, r.err, next.ID)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("AppendOnce did not finish after the cleaner committed")
|
||||
}
|
||||
}
|
||||
|
||||
// A request holding a key row must not make the cleaner wait: it skips the
|
||||
// row and deletes it on a later run.
|
||||
func TestCleanerSkipsRowLockedByRequest(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
|
||||
expire(t)
|
||||
|
||||
tx, err := db.Pool.Begin(f.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000)
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("cleaner over a locked row: deleted %d, %v; want 0", n, err)
|
||||
}
|
||||
if waited := time.Since(start); waited > time.Second {
|
||||
t.Fatalf("cleaner waited %v for the locked row", waited)
|
||||
}
|
||||
if err := tx.Rollback(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000); err != nil || n != 1 {
|
||||
t.Fatalf("cleaner after release: deleted %d, %v; want 1", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The key is alive when the claim runs but deleted before the replay lookup,
|
||||
// as by a cleaner whose clock is a moment later at the TTL boundary. The
|
||||
// operation starts over and stores the new message.
|
||||
func TestAppendOnceRetriesWhenKeyVanishes(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1")
|
||||
var once sync.Once
|
||||
restore := store.SetAfterClaim(func() {
|
||||
once.Do(func() {
|
||||
if _, err := db.Pool.Exec(context.Background(), `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
defer restore()
|
||||
next := f.newMessage(t, c.ID, alice)
|
||||
if got := f.appendOnce(t, next, "k1"); got.ID != next.ID {
|
||||
t.Fatalf("got %v, want the new message %v", got.ID, next.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
for _, k := range []string{"a", "b", "c"} {
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), k)
|
||||
}
|
||||
expire(t)
|
||||
f.appendOnce(t, f.newMessage(t, c.ID, alice), "fresh")
|
||||
|
||||
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 2 {
|
||||
t.Fatalf("first batch deleted %d, %v; want 2", n, err)
|
||||
}
|
||||
if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 1 {
|
||||
t.Fatalf("second batch deleted %d, %v; want 1", n, err)
|
||||
}
|
||||
var left int
|
||||
if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 {
|
||||
t.Fatalf("%d keys left, %v; want only the fresh one", left, err)
|
||||
}
|
||||
if n := f.count(t, c.ID); n != 4 {
|
||||
t.Fatalf("cleanup removed messages: %d left, want 4", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Every request brings its own new message under one key; exactly one is
|
||||
// stored and all of them get it back. This also proves the key row and the
|
||||
// message it references are written by one statement.
|
||||
func TestConcurrentAppendOnceSameKeyStoresOneMessage(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice := f.mustUser(t)
|
||||
c := f.mustChat(t, alice)
|
||||
const requests = 20
|
||||
ids := make(chan uuid.UUID, requests)
|
||||
var wg sync.WaitGroup
|
||||
for range requests {
|
||||
m := f.newMessage(t, c.ID, alice)
|
||||
wg.Go(func() {
|
||||
got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
ids <- got.ID
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(ids)
|
||||
seen := map[uuid.UUID]int{}
|
||||
for id := range ids {
|
||||
seen[id]++
|
||||
}
|
||||
if len(seen) != 1 {
|
||||
t.Fatalf("requests got %d different messages, want 1: %v", len(seen), seen)
|
||||
}
|
||||
if n := f.count(t, c.ID); n != 1 {
|
||||
t.Fatalf("history has %d messages, want 1", n)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
@@ -117,14 +118,90 @@ func NewMessages(pool *pgxpool.Pool) *Messages {
|
||||
return &Messages{pool: pool}
|
||||
}
|
||||
|
||||
func (r *Messages) Append(ctx context.Context, m chat.Message) error {
|
||||
_, err := r.pool.Exec(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`,
|
||||
m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert message: %w", postgres.Classify(err))
|
||||
// afterClaim runs between the claiming statement and the replay lookup of
|
||||
// AppendOnce; tests use it to delete the key in that window.
|
||||
var afterClaim func()
|
||||
|
||||
// claimSQL stores the key and the message in one statement. A key older than
|
||||
// the retention period is taken over for the new message. The key row is
|
||||
// inserted first: when it already exists and is still valid, claimed is
|
||||
// empty and so is the message insert. One statement holds the key's lock
|
||||
// only while the server executes it, never across a round trip to a replica
|
||||
// that might die in between.
|
||||
const claimSQL = `
|
||||
WITH claimed AS (
|
||||
INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET message_id = EXCLUDED.message_id, created_at = now()
|
||||
WHERE k.created_at < now() - $5::interval
|
||||
RETURNING message_id)
|
||||
INSERT INTO messages (chat_id, id, sender_id, text, created_at)
|
||||
SELECT $2, message_id, $1, $6, $7 FROM claimed`
|
||||
|
||||
// AppendOnce stores m under key unless the sender already used the key in
|
||||
// this chat within ttl; it returns the message the key is bound to, m itself
|
||||
// or the earlier one. Deciding whether the earlier message answers the same
|
||||
// request is the caller's job.
|
||||
//
|
||||
// The lookup of an existing key is a second statement: within the claiming
|
||||
// one, a plain SELECT would see the snapshot taken before a concurrent
|
||||
// request committed the key. Between the two statements the cleaner may
|
||||
// delete a key at the ttl boundary; the operation then starts over once,
|
||||
// which stores m under the now free key.
|
||||
func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) {
|
||||
for range 2 {
|
||||
tag, err := r.pool.Exec(ctx, claimSQL,
|
||||
m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt)
|
||||
if err != nil {
|
||||
return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err))
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
return m, nil
|
||||
}
|
||||
if afterClaim != nil {
|
||||
afterClaim()
|
||||
}
|
||||
var stored chat.Message
|
||||
err = r.pool.QueryRow(ctx, `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
FROM message_idempotency_keys k
|
||||
JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id
|
||||
WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`,
|
||||
m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err))
|
||||
}
|
||||
return stored, nil
|
||||
}
|
||||
return nil
|
||||
return chat.Message{}, errors.New("idempotency key vanished twice between claim and lookup")
|
||||
}
|
||||
|
||||
// DeleteExpiredKeys removes up to limit keys older than ttl. SKIP LOCKED lets
|
||||
// replicas cleaning at the same time, and requests holding a key, pass each
|
||||
// other instead of waiting.
|
||||
func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
DELETE FROM message_idempotency_keys
|
||||
WHERE (user_id, chat_id, key) IN (
|
||||
SELECT user_id, chat_id, key FROM message_idempotency_keys
|
||||
WHERE created_at < now() - $1::interval
|
||||
ORDER BY created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED)`,
|
||||
interval(ttl), limit)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err))
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// interval renders d for a ::interval parameter.
|
||||
func interval(d time.Duration) string {
|
||||
return fmt.Sprintf("%d milliseconds", d.Milliseconds())
|
||||
}
|
||||
|
||||
// ListByChat is the history query; scripts/explain-history.sql runs the same
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
@@ -81,6 +82,12 @@ func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message
|
||||
return m
|
||||
}
|
||||
|
||||
// appendErr stores m under a key of its own, as every send now carries one.
|
||||
func (f fixture) appendErr(m chat.Message) error {
|
||||
_, err := f.messages.AppendOnce(f.ctx, m, m.ID.String(), time.Hour)
|
||||
return err
|
||||
}
|
||||
|
||||
func TestChatCreateThenGetForMember(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
alice, bob := f.mustUser(t), f.mustUser(t)
|
||||
@@ -199,7 +206,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) {
|
||||
c := f.mustChat(t, alice)
|
||||
first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice)
|
||||
for _, m := range []chat.Message{third, first, second} {
|
||||
if err := f.messages.Append(f.ctx, m); err != nil {
|
||||
if err := f.appendErr(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -221,13 +228,13 @@ func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) {
|
||||
for range total {
|
||||
m := f.newMessage(t, c.ID, alice)
|
||||
expected = append(expected, m.ID)
|
||||
if err := f.messages.Append(f.ctx, m); err != nil {
|
||||
if err := f.appendErr(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Another chat's history must not leak into this one.
|
||||
other := f.mustChat(t, alice)
|
||||
if err := f.messages.Append(f.ctx, f.newMessage(t, other.ID, alice)); err != nil {
|
||||
if err := f.appendErr(f.newMessage(t, other.ID, alice)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, limit := range []int{1, 7, 100} {
|
||||
@@ -269,7 +276,7 @@ func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) {
|
||||
gap := ident.New() // never stored, but sorts between a and b
|
||||
b := f.newMessage(t, c.ID, alice)
|
||||
for _, m := range []chat.Message{a, b} {
|
||||
if err := f.messages.Append(f.ctx, m); err != nil {
|
||||
if err := f.appendErr(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -291,7 +298,7 @@ func TestConcurrentAppendsKeepEveryMessage(t *testing.T) {
|
||||
for range clients {
|
||||
wg.Go(func() {
|
||||
for range perClient {
|
||||
if err := f.messages.Append(f.ctx, f.newMessage(t, c.ID, alice)); err != nil {
|
||||
if err := f.appendErr(f.newMessage(t, c.ID, alice)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"messenger/internal/platform/apperr"
|
||||
"messenger/internal/platform/postgres"
|
||||
"messenger/internal/testenv"
|
||||
"messenger/migrations"
|
||||
)
|
||||
|
||||
var db *testenv.DB
|
||||
@@ -54,10 +56,14 @@ func TestMigrateIsRepeatable(t *testing.T) {
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 1 {
|
||||
t.Fatalf("schema version = %d, want 1", version)
|
||||
files, err := fs.Glob(migrations.FS, "*.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, table := range []string{"users", "chats", "chat_members", "messages"} {
|
||||
if version != int64(len(files)) {
|
||||
t.Fatalf("schema version = %d, want %d (one per migration file)", version, len(files))
|
||||
}
|
||||
for _, table := range []string{"users", "chats", "chat_members", "messages", "message_idempotency_keys"} {
|
||||
var exists bool
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- +goose Up
|
||||
-- One row per Idempotency-Key a user has sent to one chat. The contract scope
|
||||
-- is user + method + chat path + key; only POST /chats/{chat_id}/messages
|
||||
-- accepts keys, so (user_id, chat_id) already fixes the method and the path.
|
||||
-- The primary key is what makes concurrent retries through different
|
||||
-- replicas safe: the second insert of a key waits for the first one's commit.
|
||||
-- The row references the message instead of storing a response: messages
|
||||
-- are immutable, so a replay is answered from the source of truth.
|
||||
-- The table only adds to the schema: the previous API version keeps working
|
||||
-- while a rolling update runs this migration.
|
||||
CREATE TABLE message_idempotency_keys (
|
||||
user_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
key text NOT NULL,
|
||||
message_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, key),
|
||||
FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id)
|
||||
);
|
||||
|
||||
-- Serves the periodic removal of keys older than the retention period.
|
||||
CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE message_idempotency_keys;
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// client is a minimal typed HTTP client for the messenger API.
|
||||
@@ -33,6 +35,21 @@ func (c *client) post(path, token string, body, out any) error {
|
||||
return c.do(req, token, http.StatusCreated, out)
|
||||
}
|
||||
|
||||
// postMessage sends one message under a fresh Idempotency-Key.
|
||||
func (c *client) postMessage(chatID, token, text string, out any) error {
|
||||
raw, err := json.Marshal(map[string]string{"text": text})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Idempotency-Key", uuid.NewString())
|
||||
return c.do(req, token, http.StatusCreated, out)
|
||||
}
|
||||
|
||||
// login opens a session with Basic credentials and returns its bearer token.
|
||||
func (c *client) login(username, password string) (string, error) {
|
||||
req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil)
|
||||
|
||||
@@ -174,8 +174,7 @@ func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (m
|
||||
<-start
|
||||
for m := range perClient {
|
||||
var msg chat.MessageResponse
|
||||
body := map[string]string{"text": fmt.Sprintf("client %d message %d", n, m)}
|
||||
err := own.post("/api/v1/chats/"+chatID+"/messages", token, body, &msg)
|
||||
err := own.postMessage(chatID, token, fmt.Sprintf("client %d message %d", n, m), &msg)
|
||||
mu.Lock()
|
||||
if err != nil {
|
||||
failed++
|
||||
|
||||
Reference in New Issue
Block a user