From b323f2dab9a04583510613c48dc07a8136e62c90 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:17:15 +0300 Subject: [PATCH 1/6] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=B5=D1=81?= =?UTF-8?q?=D0=B8=D1=82=D0=B5=20=D1=80=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8E=20lab3=20=D0=B8=20=D1=81=D0=BE=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=82=D0=B5=20=D1=81=D0=B5=D1=80=D0=B8=D0=B8?= =?UTF-8?q?=20=D0=BD=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BE=D1=87=D0=BD=D1=8B?= =?UTF-8?q?=D1=85=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5=D1=80=D0=B8=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 + .gitattributes | 7 + .github/workflows/app.yml | 57 ++ .gitignore | 2 + Dockerfile | 38 ++ Makefile | 68 +++ cmd/migrate/main.go | 49 ++ cmd/server/main.go | 125 +++++ cmd/server/main_test.go | 96 ++++ compose.bench.yaml | 47 ++ compose.yaml | 158 ++++++ go.mod | 107 ++++ go.sum | 304 ++++++++++ infra/haproxy/haproxy.cfg | 74 +++ infra/valkey/valkey.conf | 19 + internal/app/app.go | 81 +++ internal/app/app_test.go | 522 ++++++++++++++++++ internal/app/auth_test.go | 176 ++++++ internal/app/failure_test.go | 101 ++++ internal/app/idempotency_test.go | 153 +++++ internal/auth/handler.go | 89 +++ internal/auth/main_test.go | 33 ++ internal/auth/middleware.go | 48 ++ internal/auth/repository.go | 18 + internal/auth/service.go | 64 +++ internal/auth/service_test.go | 156 ++++++ internal/auth/session.go | 39 ++ internal/auth/store/valkey.go | 64 +++ internal/chat/chat.go | 86 +++ internal/chat/chat_test.go | 88 +++ internal/chat/cleanup.go | 30 + internal/chat/handler.go | 209 +++++++ internal/chat/message.go | 50 ++ internal/chat/message_test.go | 60 ++ internal/chat/repository.go | 36 ++ internal/chat/service.go | 139 +++++ internal/chat/service_test.go | 391 +++++++++++++ internal/chat/store/idempotency_test.go | 228 ++++++++ internal/chat/store/postgres.go | 205 +++++++ internal/chat/store/postgres_test.go | 320 +++++++++++ internal/health/handler.go | 77 +++ internal/platform/apperr/apperr.go | 54 ++ internal/platform/apperr/apperr_test.go | 49 ++ internal/platform/config/config.go | 187 +++++++ internal/platform/config/config_test.go | 177 ++++++ internal/platform/httpx/decode.go | 70 +++ internal/platform/httpx/engine.go | 50 ++ internal/platform/httpx/httpx_test.go | 409 ++++++++++++++ internal/platform/httpx/identity.go | 42 ++ internal/platform/httpx/middleware.go | 131 +++++ internal/platform/httpx/respond.go | 87 +++ internal/platform/ident/ident.go | 56 ++ internal/platform/ident/ident_test.go | 84 +++ internal/platform/pagination/cursor.go | 63 +++ internal/platform/pagination/cursor_test.go | 71 +++ internal/platform/pagination/page.go | 41 ++ internal/platform/pagination/page_test.go | 60 ++ internal/platform/password/password.go | 74 +++ internal/platform/password/password_test.go | 78 +++ internal/platform/postgres/postgres.go | 143 +++++ internal/platform/postgres/postgres_test.go | 212 +++++++ internal/platform/valkey/valkey.go | 49 ++ internal/platform/valkey/valkey_test.go | 57 ++ internal/server/server.go | 83 +++ internal/server/server_test.go | 169 ++++++ internal/testenv/testenv.go | 181 ++++++ internal/testenv/testenv_test.go | 29 + internal/user/handler.go | 70 +++ internal/user/main_test.go | 24 + internal/user/repository.go | 18 + internal/user/service.go | 88 +++ internal/user/service_test.go | 189 +++++++ internal/user/store/postgres.go | 74 +++ internal/user/user.go | 75 +++ internal/user/user_test.go | 78 +++ migrations/00001_init.sql | 42 ++ migrations/00002_message_idempotency_keys.sql | 17 + migrations/embed.go | 8 + requirements-bench.txt | 2 + scripts/api-instances.sh | 30 + scripts/bench/failover.sh | 56 ++ scripts/bench/lib.sh | 77 +++ scripts/bench/plot.py | 336 +++++++++++ scripts/bench/rollout-under-load.sh | 50 ++ scripts/bench/run-series.sh | 88 +++ scripts/bench/series/A.conf | 6 + scripts/bench/series/Aprime.conf | 5 + scripts/bench/series/B.conf | 5 + scripts/bench/series/C.conf | 10 + scripts/bench/series/control.conf | 6 + scripts/explain-history.sql | 75 +++ scripts/init-secrets.sh | 42 ++ scripts/instances.sh | 9 + scripts/restart.cmd | 5 + scripts/restart.sh | 7 + scripts/rollout.sh | 44 ++ scripts/stop-one.cmd | 5 + scripts/stop-one.sh | 27 + tools/concurrency/client.go | 135 +++++ tools/concurrency/main.go | 322 +++++++++++ tools/concurrency/main_test.go | 89 +++ tools/loadgen/client.go | 159 ++++++ tools/loadgen/main.go | 343 ++++++++++++ tools/loadgen/main_test.go | 135 +++++ tools/loadgen/run.go | 49 ++ tools/loadgen/scrape.go | 139 +++++ tools/loadgen/seed.go | 132 +++++ tools/loadgen/stats.go | 205 +++++++ tools/loadgen/stats_test.go | 148 +++++ 109 files changed, 10747 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 cmd/migrate/main.go create mode 100644 cmd/server/main.go create mode 100644 cmd/server/main_test.go create mode 100644 compose.bench.yaml create mode 100644 compose.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/valkey/valkey.conf create mode 100644 internal/app/app.go create mode 100644 internal/app/app_test.go create mode 100644 internal/app/auth_test.go create mode 100644 internal/app/failure_test.go create mode 100644 internal/app/idempotency_test.go create mode 100644 internal/auth/handler.go create mode 100644 internal/auth/main_test.go create mode 100644 internal/auth/middleware.go create mode 100644 internal/auth/repository.go create mode 100644 internal/auth/service.go create mode 100644 internal/auth/service_test.go create mode 100644 internal/auth/session.go create mode 100644 internal/auth/store/valkey.go create mode 100644 internal/chat/chat.go create mode 100644 internal/chat/chat_test.go create mode 100644 internal/chat/cleanup.go create mode 100644 internal/chat/handler.go create mode 100644 internal/chat/message.go create mode 100644 internal/chat/message_test.go create mode 100644 internal/chat/repository.go create mode 100644 internal/chat/service.go create mode 100644 internal/chat/service_test.go create mode 100644 internal/chat/store/idempotency_test.go create mode 100644 internal/chat/store/postgres.go create mode 100644 internal/chat/store/postgres_test.go create mode 100644 internal/health/handler.go create mode 100644 internal/platform/apperr/apperr.go create mode 100644 internal/platform/apperr/apperr_test.go create mode 100644 internal/platform/config/config.go create mode 100644 internal/platform/config/config_test.go create mode 100644 internal/platform/httpx/decode.go create mode 100644 internal/platform/httpx/engine.go create mode 100644 internal/platform/httpx/httpx_test.go create mode 100644 internal/platform/httpx/identity.go create mode 100644 internal/platform/httpx/middleware.go create mode 100644 internal/platform/httpx/respond.go create mode 100644 internal/platform/ident/ident.go create mode 100644 internal/platform/ident/ident_test.go create mode 100644 internal/platform/pagination/cursor.go create mode 100644 internal/platform/pagination/cursor_test.go create mode 100644 internal/platform/pagination/page.go create mode 100644 internal/platform/pagination/page_test.go create mode 100644 internal/platform/password/password.go create mode 100644 internal/platform/password/password_test.go create mode 100644 internal/platform/postgres/postgres.go create mode 100644 internal/platform/postgres/postgres_test.go create mode 100644 internal/platform/valkey/valkey.go create mode 100644 internal/platform/valkey/valkey_test.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go create mode 100644 internal/testenv/testenv.go create mode 100644 internal/testenv/testenv_test.go create mode 100644 internal/user/handler.go create mode 100644 internal/user/main_test.go create mode 100644 internal/user/repository.go create mode 100644 internal/user/service.go create mode 100644 internal/user/service_test.go create mode 100644 internal/user/store/postgres.go create mode 100644 internal/user/user.go create mode 100644 internal/user/user_test.go create mode 100644 migrations/00001_init.sql create mode 100644 migrations/00002_message_idempotency_keys.sql create mode 100644 migrations/embed.go create mode 100644 requirements-bench.txt create mode 100755 scripts/api-instances.sh create mode 100755 scripts/bench/failover.sh create mode 100644 scripts/bench/lib.sh create mode 100755 scripts/bench/plot.py create mode 100755 scripts/bench/rollout-under-load.sh create mode 100755 scripts/bench/run-series.sh create mode 100644 scripts/bench/series/A.conf create mode 100644 scripts/bench/series/Aprime.conf create mode 100644 scripts/bench/series/B.conf create mode 100644 scripts/bench/series/C.conf create mode 100644 scripts/bench/series/control.conf create mode 100644 scripts/explain-history.sql create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/instances.sh create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100755 scripts/rollout.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh create mode 100644 tools/concurrency/client.go create mode 100644 tools/concurrency/main.go create mode 100644 tools/concurrency/main_test.go create mode 100644 tools/loadgen/client.go create mode 100644 tools/loadgen/main.go create mode 100644 tools/loadgen/main_test.go create mode 100644 tools/loadgen/run.go create mode 100644 tools/loadgen/scrape.go create mode 100644 tools/loadgen/seed.go create mode 100644 tools/loadgen/stats.go create mode 100644 tools/loadgen/stats_test.go diff --git a/.dockerignore b/.dockerignore index bc16e556..95cf4c12 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store +compose*.yaml +bench-out +bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2cc26e76 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Files read inside Linux containers or by sh must keep LF on Windows +# checkouts with core.autocrlf=true. +*.sh text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.cmd text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..936f2ec4 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,57 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +defaults: + run: + # Explicit bash turns on `-eo pipefail`, so a failing check behind `| tee` fails the job. + shell: bash +env: + BASE_URL: http://localhost:8080 +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - run: go vet ./... + - run: go test -race -count=1 ./... + e2e: + runs-on: ubuntu-latest + needs: unit + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: make secrets + - run: docker compose up -d --build --wait + - run: mkdir -p evidence-ci + - name: Public smoke + run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt + - name: Persistence across a restart of the API and both stores + run: make test-persistence PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/restart.sh | tee evidence-ci/persistence.txt + - name: Failover of one replica behind the balancer + run: | + make test-failover PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/stop-one.sh | tee evidence-ci/failover.txt + docker compose start api + docker compose up -d --wait + - name: Concurrency and shared idempotency across replicas + run: go run ./tools/concurrency --base-url "$BASE_URL" --output evidence-ci/concurrency.json + - name: Container logs + if: always() + run: docker compose logs > evidence-ci/compose.log 2>&1 || true + - name: Cleanup + # The CI stand is disposable, so its volumes go too. + if: always() + run: docker compose down -v || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: evidence + path: evidence-ci diff --git a/.gitignore b/.gitignore index b5a71896..94c5da0a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Raw output of local benchmark runs; curated results go to evidence/. +bench-out/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2a0b1902 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# Modules in their own layer and a persistent build cache keep the rebuild of +# a rolling update short. +FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download && go mod verify +COPY cmd ./cmd +COPY internal ./internal +COPY migrations ./migrations +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate + +# The load generator gets an image of its own; the API image does not carry it. +FROM build AS loadgen-build +COPY tools ./tools +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen + +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen +RUN adduser -D -H -u 10001 app +COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen +USER app +ENTRYPOINT ["/usr/local/bin/loadgen"] + +# Last, so that `build: .` builds it; it also runs the migrate service. +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 +RUN adduser -D -H -u 10001 app +COPY --from=build /out/server /out/migrate /usr/local/bin/ +USER app +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab +EXPOSE 8080 +# The start interval lets a rolling update see a new replica healthy within +# about a second. +HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ + CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 +ENTRYPOINT ["/usr/local/bin/server"] diff --git a/Makefile b/Makefile index 5d86d171..21b7346d 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,71 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --------------------------------------------------------------------------- +# Application targets (Go implementation). The targets above are the skeleton. +# --------------------------------------------------------------------------- +EVIDENCE_DIR ?= evidence + +.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout bench plot failover-load rollout-load + +build: + go build -o bin/server ./cmd/server + +vet: + go vet ./... + +# The tests start disposable PostgreSQL and Valkey containers: Docker must run. +unit: + go test -race -count=1 ./... + +secrets: + sh scripts/init-secrets.sh + +up: + docker compose up -d --build --wait + +# Stops and removes the containers; volumes and therefore data are kept. +down: + docker compose down + +# Deletes the volumes too: every user, chat, message and session of the stand. +destroy: + docker compose down -v + +logs: + docker compose logs + +# Which container answers under which X-Instance-Id. +instances: + sh scripts/instances.sh + +# Replaces every API replica without downtime; all of them must be running. +rollout: + sh scripts/rollout.sh + +# Track 1. Every run starts from empty volumes, so the stand's data is lost. +# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200 +SERIES ?= scripts/bench/series/A.conf +bench: + sh scripts/bench/run-series.sh $(SERIES) + +# Needs matplotlib from requirements-bench.txt: make plot PYTHON=. +plot: + $(PYTHON) scripts/bench/plot.py evidence/track1/raw evidence/track1 + +# Level 4 under load: one replica stopped (KIND=stop) or killed (KIND=kill), +# and the rolling update. +KIND ?= stop +REPLICA ?= 1 +failover-load: + sh scripts/bench/failover.sh $(KIND) $(REPLICA) + +rollout-load: + sh scripts/bench/rollout-under-load.sh + +concurrency: + go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" + +explain: + docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..851b2e85 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,49 @@ +// Command migrate applies the embedded schema migrations once and exits, so +// the API replicas never race each other over the schema. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" +) + +// poolSize covers the goose session lock and the migration itself. +const poolSize = 2 + +func main() { + cfg, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + log := cfg.NewLogger() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg); err != nil { + log.Error("migration failed", "error", err.Error()) + os.Exit(1) + } +} + +func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, poolSize) + if err != nil { + return err + } + defer pool.Close() + applied, err := postgres.Migrate(ctx, pool) + if err != nil { + return err + } + log.Info("migrations applied", "applied_migrations", applied) + return nil +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 00000000..9d4f4b94 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,125 @@ +// Command server starts the messenger API over PostgreSQL and Valkey. +package main + +import ( + "context" + "expvar" + "fmt" + "log/slog" + "net" + "os" + "os/signal" + "runtime" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/app" + "messenger/internal/chat" + "messenger/internal/platform/config" + "messenger/internal/platform/ident" + "messenger/internal/platform/postgres" + "messenger/internal/platform/valkey" + "messenger/internal/server" +) + +const keyCleanupEvery = 10 * time.Minute + +func main() { + cfg, err := config.Load(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + + alias := ident.NewAlias() + log := cfg.NewLogger().With("instance", alias) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg, alias); err != nil { + log.Error("server stopped with error", "error", err.Error()) + os.Exit(1) + } +} + +// run connects the dependencies before it listens, so the port accepts +// connections only once they answer. +func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns) + if err != nil { + return err + } + defer pool.Close() + sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL) + if err != nil { + return err + } + defer sessions.Close() + + ln, err := net.Listen("tcp", cfg.Addr()) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.Addr(), err) + } + defer ln.Close() + log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(), + "gomaxprocs", runtime.GOMAXPROCS(0), "db_max_conns", cfg.DBMaxConns, + "drain_delay", cfg.DrainDelay.String(), "shutdown_timeout", cfg.ShutdownTimeout.String(), + "session_ttl", cfg.SessionTTL.String(), "idempotency_ttl", cfg.IdempotencyTTL.String()) + a := app.New(app.Deps{ + Log: log, + DB: pool, + Sessions: sessions, + SessionTTL: cfg.SessionTTL, + MaxBodyBytes: cfg.MaxBodyBytes, + InstanceID: alias, + IdempotencyTTL: cfg.IdempotencyTTL, + }) + var debugLn net.Listener + if cfg.DebugAddr != "" { + if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil { + return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) + } + publishDebugVars(alias, pool) + } + + ctx, cancel := context.WithCancel(ctx) + var background sync.WaitGroup + background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if debugLn != nil { + background.Go(func() { + if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { + log.Error("debug listener stopped", "error", err.Error()) + } + }) + } + + err = server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) + cancel() + background.Wait() + return err +} + +// publishDebugVars adds to expvar what the load tests read from a replica. +func publishDebugVars(alias string, pool *pgxpool.Pool) { + expvar.NewString("instance").Set(alias) + expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) })) + expvar.Publish("pgxpool", expvar.Func(func() any { + s := pool.Stat() + return map[string]any{ + "acquire_count": s.AcquireCount(), + "acquire_duration_ms": s.AcquireDuration().Milliseconds(), + "acquired_conns": s.AcquiredConns(), + "idle_conns": s.IdleConns(), + "total_conns": s.TotalConns(), + "max_conns": s.MaxConns(), + "empty_acquire_count": s.EmptyAcquireCount(), + "empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(), + "canceled_acquire_count": s.CanceledAcquireCount(), + "new_conns_count": s.NewConnsCount(), + } + })) +} diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go new file mode 100644 index 00000000..296607aa --- /dev/null +++ b/cmd/server/main_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "os" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +// A start-up failure after the background work began must still end the +// process with an error, not leave it waiting for a signal. +func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) { + if err := runWithBusyDebugAddr(t, 0); err == nil { + t.Fatal("run succeeded on a busy debug address") + } +} + +func TestFailedStartReleasesTheAPIPort(t *testing.T) { + probe, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := probe.Addr().(*net.TCPAddr).Port + probe.Close() + + if err := runWithBusyDebugAddr(t, port); err == nil { + t.Fatal("run succeeded on a busy debug address") + } + ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + t.Fatalf("the API port is still taken after the failed start: %v", err) + } + ln.Close() +} + +func runWithBusyDebugAddr(t *testing.T, port int) error { + t.Helper() + busy, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer busy.Close() + cfg := config.Config{ + Host: "127.0.0.1", + Port: port, + ShutdownTimeout: time.Second, + MaxBodyBytes: 1024, + DatabaseURL: db.DSN, + SessionStoreURL: kv.URL, + SessionTTL: time.Hour, + IdempotencyTTL: time.Hour, + DBMaxConns: 2, + DebugAddr: busy.Addr().String(), + } + done := make(chan error, 1) + go func() { + done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias") + }() + select { + case err := <-done: + return err + case <-time.After(10 * time.Second): + t.Fatal("run hung instead of returning the listen error") + return nil + } +} diff --git a/compose.bench.yaml b/compose.bench.yaml new file mode 100644 index 00000000..f61d47e7 --- /dev/null +++ b/compose.bench.yaml @@ -0,0 +1,47 @@ +# Load-test override of compose.yaml, used only by scripts/bench: every value +# an experiment varies, and the load generator. Defaults equal compose.yaml. +services: + lb: + environment: + LB_QUIET: ${LB_QUIET:-0} + API_MAXCONN: ${API_MAXCONN:-20} + deploy: + resources: + limits: + cpus: "${LB_CPUS:-0.5}" + + api: + environment: + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} + APP_DEBUG_ADDR: ":9090" + deploy: + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-256M} + + postgres: + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEMORY:-512M} + + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab4 + environment: + GOMEMLIMIT: 900MiB + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-2.0}" + memory: 1G diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..5b3e7b7e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,158 @@ +# Local stand: HAProxy in front of interchangeable API replicas over shared +# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data +# network has no way out. The load-test knobs are in compose.bench.yaml. +name: lab4 + +services: + lb: + image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "${PUBLISH:-127.0.0.1:8080}:8080" + - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" + networks: + - edge + healthcheck: + # Through the balancer: `up --wait` returns once a replica is reachable. + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + api: + build: . + image: mtusi-messenger:lab4 + env_file: + - path: .env + required: false + environment: + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_STORE_URL_FILE: /run/secrets/session_store_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + # Go does not read the cgroup memory limit. + GOMEMLIMIT: 230MiB + secrets: + - database_url + - session_store_url + networks: + - edge + - data + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_healthy + # Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL. + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 256M + + migrate: + image: mtusi-messenger:lab4 + entrypoint: ["/usr/local/bin/migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + networks: + - data + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + # The image's check probes the API port, which migrate never opens. + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + postgres: + image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 + # 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql. + command: ["postgres", "-c", "max_connections=150"] + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + # PostgreSQL 18 keeps PGDATA in a versioned directory below this one. + volumes: + - pgdata:/var/lib/postgresql + networks: + - data + healthcheck: + # Over TCP: the first initialisation runs a temporary socket-only server. + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + valkey: + image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + secrets: + - valkey_auth + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + - valkeydata:/data + networks: + - data + healthcheck: + # valkey-cli exits 0 even on NOAUTH, so the reply itself is checked. + test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.5" + # maxmemory 256mb (valkey.conf) plus the server's own overhead. + memory: 320M + +volumes: + pgdata: + valkeydata: + +networks: + edge: {} + data: + internal: true + +secrets: + database_url: + file: ./secrets/database_url + session_store_url: + file: ./secrets/session_store_url + postgres_password: + file: ./secrets/postgres_password + valkey_auth: + file: ./secrets/valkey_auth.conf diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..9c6fb13e --- /dev/null +++ b/go.mod @@ -0,0 +1,107 @@ +module messenger + +go 1.27.1 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/pressly/goose/v3 v3.27.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/testcontainers/testcontainers-go v0.39.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 + github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 + golang.org/x/crypto v0.57.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/creack/pty v1.1.24 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.3.3+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/stretchr/testify v1.12.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..fb12767b --- /dev/null +++ b/go.sum @@ -0,0 +1,304 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.27.0 h1:/D30gVTuQhu0WsNZYbJi4DMOsx1lNq+6SkLe+Wp59BM= +github.com/pressly/goose/v3 v3.27.0/go.mod h1:3ZBeCXqzkgIRvrEMDkYh1guvtoJTU5oMMuDdkutoM78= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= +github.com/testcontainers/testcontainers-go v0.39.0/go.mod h1:qmHpkG7H5uPf/EvOORKvS6EuDkBUPE3zpVGaH9NL7f8= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/valkey-io/valkey-go v1.0.59 h1:W67Z0UY+Qqk3k8NKkFCFlM3X4yQUniixl7dSJAch2Qo= +github.com/valkey-io/valkey-go v1.0.59/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ= +modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..129badc3 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,74 @@ +# The single entry point: round-robin over the API replicas found through +# Docker DNS, with active readiness checks and no client stickiness. + +global + log stdout format raw local0 info + maxconn 20000 + # The container has at most one CPU; by default HAProxy starts a thread + # per host CPU and fights the quota. + nbthread 1 + +defaults + mode http + log global + option dontlognull + timeout connect 1s + timeout client 30s + # Above the API's 5 s request deadline: the API answers a slow request + # itself, this only cuts off a hung replica. + timeout server 7s + timeout http-request 10s + timeout http-keep-alive 10s + timeout check 2s + # Below the 500 ms latency objective: excess load gets a fast 503. + timeout queue 300ms + # Re-sent only when nothing reached the API: a timeout or an empty + # response may follow a committed write. + retries 2 + option redispatch 1 + retry-on conn-failure + # Errors produced by the balancer itself use the contract's envelope. + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + +# `api` resolves to every running replica, re-resolved every second; an +# address that disappears leaves at once. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + hold obsolete 0s + accepted_payload_size 8192 + +frontend entry + bind :8080 + option httplog + unique-id-format %[uuid(4),regsub(-,,g)] + # Replaces any client value; the API keeps it, so one id spans both logs. + http-request set-header X-Request-Id %[unique-id] + capture response header X-Instance-Id len 32 + capture response header X-Request-Id len 40 + http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 } + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. + # maxconn 20: the replica's 10 pooled database connections plus requests + # that need no database; the rest waits in the queue. + default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 + # Up to five replicas, doubled while docker rollout replaces them. + server-template api 10 api:8080 + +listen stats + bind :8404 + no log + stats enable + stats uri / + stats refresh 2s diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..d7e96bff --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,19 @@ +# Valkey is the only store of user sessions, so it is their source of truth. +# Every write is fsynced before the reply: an acknowledged login survives a +# crash or a restart. Only logins and logouts write; session lookups are reads +# and pay nothing for this. +appendonly yes +appendfsync always +dir /data + +# The append-only file already persists everything; RDB snapshots are off. +save "" + +# Sessions are never evicted to free memory: once maxmemory is reached the +# store refuses new sessions (login answers 503) instead of silently logging +# users out. A session key takes about a hundred bytes. +maxmemory 256mb +maxmemory-policy noeviction + +# requirepass comes from the compose secret, never from the repository. +include /run/secrets/valkey_auth diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 00000000..720a9ae5 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,81 @@ +// Package app is the composition root: it builds every module with its +// stores and mounts them on one HTTP engine. Both the server binary and the +// integration tests start from here. +package app + +import ( + "context" + "log/slog" + "time" + + "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/chat" + chatstore "messenger/internal/chat/store" + "messenger/internal/health" + "messenger/internal/platform/httpx" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +type App struct { + Engine *gin.Engine + Health *health.Handler + // Chats is exposed for the background cleanup of idempotency keys. + Chats *chat.Service +} + +// Deps are the shared resources the modules are built on; their lifetime is +// owned by the caller. +type Deps struct { + Log *slog.Logger + DB *pgxpool.Pool + Sessions *redis.Client + SessionTTL time.Duration + MaxBodyBytes int64 + // RequestTimeout bounds every API request; zero means the default. + RequestTimeout time.Duration + InstanceID string + // IdempotencyTTL zero means the contract's 24 hours. + IdempotencyTTL time.Duration +} + +// defaultRequestTimeout stays below the server's 10s write timeout, so a +// request that gives up on a dependency can still write its 503. +const defaultRequestTimeout = 5 * time.Second + +const defaultIdempotencyTTL = 24 * time.Hour + +func New(d Deps) *App { + users := user.NewService(userstore.NewUsers(d.DB)) + keyTTL := d.IdempotencyTTL + if keyTTL == 0 { + keyTTL = defaultIdempotencyTTL + } + chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL) + + engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) + probes := health.New( + health.Check{Name: "postgres", Probe: d.DB.Ping}, + health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, + ) + probes.Register(engine) + + sessions := auth.NewService(users, authstore.NewSessions(d.Sessions), d.SessionTTL) + + timeout := d.RequestTimeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + public := engine.Group("/api/v1", httpx.Timeout(timeout)) + protected := public.Group("", auth.RequireSession(sessions)) + auth.NewHandler(sessions).Register(public, protected) + user.NewHandler(users).Register(public, protected) + chat.NewHandler(chats).Register(protected) + + return &App{Engine: engine, Health: probes, Chats: chats} +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 00000000..02f8a33b --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,522 @@ +package app_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/app" + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/testenv" + "messenger/internal/user" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +const testPassword = "correct-horse-battery" + +const testInstance = "00000000000000a1" + +// errorBody mirrors the contract error envelope. +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type response struct { + status int + requestID string + challenge string + raw []byte +} + +// tester drives the assembled application in-process. +type tester struct { + t *testing.T + a *app.App +} + +// account is a registered user with a live session. +type account struct { + user.Response + token string +} + +// sessionBody mirrors the contract Session schema. +type sessionBody struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +func newTester(t *testing.T) *tester { + t.Helper() + return newTesterWithTTL(t, time.Hour) +} + +// newTesterWithTTL assembles the application over empty stores. +func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester { + t.Helper() + db.Reset(t) + kv.Reset(t) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + return &tester{t: t, a: app.New(app.Deps{ + Log: log, + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: ttl, + MaxBodyBytes: 64 * 1024, + InstanceID: testInstance, + })} +} + +// do sends one request, authenticated with the bearer token when it is set. +// A string body is sent verbatim, anything else is JSON-encoded; both get a +// JSON content type. Every response is checked for X-Request-Id, error +// bodies for the contract envelope, and every 401 for the challenge of the +// scheme the route expects. +func (ts *tester) do(method, path, token string, body any, headers map[string]string) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, encode(ts.t, body)) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + + res := response{ + status: rec.Code, + requestID: rec.Header().Get("X-Request-Id"), + challenge: rec.Header().Get("WWW-Authenticate"), + raw: rec.Body.Bytes(), + } + if res.requestID == "" { + ts.t.Fatalf("%s %s: missing X-Request-Id", method, path) + } + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + ts.t.Fatalf("%s %s: X-Instance-Id = %q, want %q", method, path, got, testInstance) + } + if res.status >= 400 { + var eb errorBody + if err := json.Unmarshal(res.raw, &eb); err != nil { + ts.t.Fatalf("%s %s: error body is not the contract envelope: %v: %s", method, path, err, res.raw) + } + if eb.Error.RequestID != res.requestID || eb.Error.Code == "" || eb.Error.Message == "" { + ts.t.Fatalf("%s %s: bad error envelope: %s", method, path, res.raw) + } + } + if res.status == http.StatusUnauthorized { + scheme := "Bearer " + if path == "/api/v1/auth/sessions" { + scheme = "Basic " + } + if !strings.HasPrefix(res.challenge, scheme) { + ts.t.Fatalf("%s %s: 401 challenge = %q, want scheme %q", method, path, res.challenge, scheme) + } + } + return res +} + +func encode(t *testing.T, body any) io.Reader { + t.Helper() + switch b := body.(type) { + case nil: + return nil + case string: + return strings.NewReader(b) + default: + raw, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + return bytes.NewReader(raw) + } +} + +// expect asserts the status and, for errors, the contract code. +func (ts *tester) expect(res response, status int, code string) { + ts.t.Helper() + if res.status != status { + ts.t.Fatalf("status = %d, want %d; body %s", res.status, status, res.raw) + } + if code != "" { + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + if eb.Error.Code != code { + ts.t.Fatalf("code = %q, want %q; body %s", eb.Error.Code, code, res.raw) + } + } +} + +// decode parses a success body into a typed DTO and rejects unknown fields, +// which enforces the additionalProperties=false rule of the contract. +func decode[T any](t *testing.T, res response) T { + t.Helper() + var out T + dec := json.NewDecoder(bytes.NewReader(res.raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + t.Fatalf("body does not match %T: %v: %s", out, err, res.raw) + } + return out +} + +// user registers a user and logs it in. +func (ts *tester) user(username, display string) account { + ts.t.Helper() + body := map[string]string{"username": username, "display_name": display, "password": testPassword} + res := ts.do(http.MethodPost, "/api/v1/users", "", body, nil) + ts.expect(res, http.StatusCreated, "") + registered := decode[user.Response](ts.t, res) + return account{Response: registered, token: ts.login(username, testPassword).Token} +} + +func basic(username, password string) map[string]string { + raw := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + return map[string]string{"Authorization": "Basic " + raw} +} + +func (ts *tester) login(username, password string) sessionBody { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic(username, password)) + ts.expect(res, http.StatusCreated, "") + return decode[sessionBody](ts.t, res) +} + +func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats", token, map[string]any{"title": title, "member_ids": members}, nil) + ts.expect(res, http.StatusCreated, "") + return decode[chat.ChatResponse](ts.t, res) +} + +func (ts *tester) message(token, chatID, text string) chat.MessageResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, freshKey()) + ts.expect(res, http.StatusCreated, "") + return decode[chat.MessageResponse](ts.t, res) +} + +func freshKey() map[string]string { + return map[string]string{"Idempotency-Key": uuid.NewString()} +} + +func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.ChatResponse]](ts.t, res) +} + +func (ts *tester) history(token, chatID, query string) pagination.PageResponse[chat.MessageResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats/"+chatID+"/messages"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.MessageResponse]](ts.t, res) +} + +func cursorQuery(limit int, cursor *string) string { + q := fmt.Sprintf("?limit=%d", limit) + if cursor != nil { + q += "&cursor=" + url.QueryEscape(*cursor) + } + return q +} + +func TestHealthEndpoints(t *testing.T) { + ts := newTester(t) + for _, path := range []string{"/health/live", "/health/ready"} { + res := ts.do(http.MethodGet, path, "", nil, nil) + ts.expect(res, http.StatusOK, "") + if strings.TrimSpace(string(res.raw)) != `{"status":"ok"}` { + t.Fatalf("%s body = %s", path, res.raw) + } + } + ts.a.Health.SetReady(false) + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") +} + +func TestRegisterUserShapeAndConflicts(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + if alice.ID == "" || alice.Username != "alice" || alice.DisplayName != "Алиса" { + t.Fatalf("user = %+v", alice) + } + if !strings.HasSuffix(alice.CreatedAt, "Z") || len(alice.CreatedAt) != len("2026-09-01T12:00:00.000Z") { + t.Fatalf("created_at = %q", alice.CreatedAt) + } + post := func(body any) response { return ts.do(http.MethodPost, "/api/v1/users", "", body, nil) } + ts.expect(post(map[string]string{"username": "alice", "display_name": "Other", "password": testPassword}), http.StatusConflict, "conflict") + ts.expect(post(map[string]string{"username": "bob", "display_name": "Bob"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "Alice", "display_name": "x"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "bob", "display_name": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": "bob", "display_name": "x", "password": "no"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(`{"username":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": 5, "display_name": "x"}), http.StatusBadRequest, "invalid_request") +} + +func TestCurrentUserIsTheTokenOwner(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + ts.user("bob", "Боб") + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got != alice.Response { + t.Fatalf("me = %+v, want %+v", got, alice.Response) + } +} + +func TestChatCreationRules(t *testing.T) { + ts := newTester(t) + alice, bob := ts.user("alice", "A"), ts.user("bob", "B") + created := ts.chat(alice.token, "Проект по ВНП", alice.ID, bob.ID) + if created.Title != "Проект по ВНП" || len(created.MemberIDs) != 2 || created.MemberIDs[0] != alice.ID || created.MemberIDs[1] != bob.ID { + t.Fatalf("chat = %+v", created) + } + unknown := "22222222-2222-4222-8222-222222222222" + invalid := map[string]any{ + "creator missing": map[string]any{"title": "t", "member_ids": []string{bob.ID}}, + "duplicate": map[string]any{"title": "t", "member_ids": []string{alice.ID, alice.ID}}, + "unknown member": map[string]any{"title": "t", "member_ids": []string{alice.ID, unknown}}, + "malformed uuid": map[string]any{"title": "t", "member_ids": []string{alice.ID, "not-a-uuid"}}, + "empty members": map[string]any{"title": "t", "member_ids": []string{}}, + "blank title": map[string]any{"title": "", "member_ids": []string{alice.ID}}, + "long title": map[string]any{"title": strings.Repeat("я", 101), "member_ids": []string{alice.ID}}, + "missing members": map[string]any{"title": "t"}, + "extra field": map[string]any{"title": "t", "member_ids": []string{alice.ID}, "owner": alice.ID}, + } + for name, body := range invalid { + res := ts.do(http.MethodPost, "/api/v1/chats", alice.token, body, nil) + if res.status != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body %s", name, res.status, res.raw) + } + } + ts.expect(ts.do(http.MethodPost, "/api/v1/chats", "", map[string]any{"title": "t", "member_ids": []string{alice.ID}}, nil), http.StatusUnauthorized, "unauthorized") +} + +func TestChatVisibilityAndListing(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + created := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + created.ID + + res := ts.do(http.MethodGet, path, bob.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[chat.ChatResponse](t, res); got.ID != created.ID || len(got.MemberIDs) != 2 { + t.Fatalf("GET chat = %+v", got) + } + ts.expect(ts.do(http.MethodGet, path, eve.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/33333333-3333-4333-8333-333333333333", alice.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/not-a-uuid", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, path, "", nil, nil), http.StatusUnauthorized, "unauthorized") + + listed := ts.chats(bob.token, "?limit=100") + if len(listed.Items) != 1 || listed.Items[0].ID != created.ID || listed.NextCursor != nil { + t.Fatalf("bob list = %+v", listed) + } + hidden := ts.do(http.MethodGet, "/api/v1/chats?limit=100", eve.token, nil, nil) + ts.expect(hidden, http.StatusOK, "") + if !strings.Contains(string(hidden.raw), `"items":[]`) || !strings.Contains(string(hidden.raw), `"next_cursor":null`) { + t.Fatalf("empty list must serialize items as [] and next_cursor as null: %s", hidden.raw) + } +} + +func TestChatListPagination(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + var expected []string + for i := 0; i < 5; i++ { + expected = append(expected, ts.chat(alice.token, fmt.Sprintf("chat %d", i), alice.ID).ID) + } + var walked []string + var cursor *string + for pages := 0; pages < 10; pages++ { + page := ts.chats(alice.token, cursorQuery(2, cursor)) + for _, c := range page.Items { + walked = append(walked, c.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walked %v, want %v", walked, expected) + } + + bob := ts.user("bob", "B") + foreign := ts.chats(alice.token, "?limit=1").NextCursor + ts.expect(ts.do(http.MethodGet, "/api/v1/chats"+cursorQuery(50, foreign), bob.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?cursor=", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?limit=abc", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessagesEndToEnd(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + + first := ts.message(alice.token, room.ID, "Привет, мир 👋") + second := ts.message(bob.token, room.ID, "Сообщение Боба") + if first.SenderID != alice.ID || second.SenderID != bob.ID { + t.Fatalf("sender must come from the bearer token: %+v %+v", first, second) + } + if first.ChatID != room.ID || first.Text != "Привет, мир 👋" { + t.Fatalf("message = %+v", first) + } + + page1 := ts.history(bob.token, room.ID, "?limit=1") + if len(page1.Items) != 1 || page1.NextCursor == nil || page1.Items[0] != first { + t.Fatalf("page1 = %+v", page1) + } + page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor)) + if len(page2.Items) != 1 || page2.NextCursor != nil || page2.Items[0] != second { + t.Fatalf("page2 = %+v", page2) + } + + post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, freshKey()) } + ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": "x", "sender_id": bob.ID}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, `{"text":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(eve.token, map[string]string{"text": "чужое"}), http.StatusNotFound, "not_found") + ts.expect(post(eve.token, map[string]string{"text": ""}), http.StatusNotFound, "not_found") + ts.expect(post("", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4000)}), http.StatusCreated, "") + + get := func(actor, query string) response { return ts.do(http.MethodGet, path+query, actor, nil, nil) } + ts.expect(get(eve.token, ""), http.StatusNotFound, "not_found") + ts.expect(get(alice.token, "?limit=0"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?limit=101"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?cursor=garbage"), http.StatusBadRequest, "invalid_request") + + other := ts.chat(alice.token, "другой", alice.ID) + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+other.ID+"/messages"+cursorQuery(50, page1.NextCursor), alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessageHistoryWalkAcrossPages(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "t", alice.ID) + var expected []string + for i := 0; i < 23; i++ { + expected = append(expected, ts.message(alice.token, room.ID, fmt.Sprintf("m%d", i)).ID) + } + var walked []string + var cursor *string + for { + page := ts.history(alice.token, room.ID, cursorQuery(5, cursor)) + for _, m := range page.Items { + walked = append(walked, m.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walk mismatch:\n%v\n%v", walked, expected) + } +} + +func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) { + ts := newTester(t) + const clients, perClient = 10, 10 + clientsList := make([]account, 0, clients) + members := make([]string, 0, clients) + for i := 0; i < clients; i++ { + c := ts.user(fmt.Sprintf("client_%d", i), "C") + clientsList = append(clientsList, c) + members = append(members, c.ID) + } + room := ts.chat(clientsList[0].token, "нагрузка", members...) + + var wg sync.WaitGroup + for n, c := range clientsList { + wg.Go(func() { + for i := 0; i < perClient; i++ { + res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, freshKey()) + if res.status != http.StatusCreated { + t.Errorf("client %d: status %d body %s", n, res.status, res.raw) + } + } + }) + } + wg.Wait() + + page := ts.history(clientsList[0].token, room.ID, "?limit=100") + if len(page.Items) != clients*perClient || page.NextCursor != nil { + t.Fatalf("got %d messages, next=%v; want %d and null", len(page.Items), page.NextCursor, clients*perClient) + } + ids := make([]string, 0, len(page.Items)) + unique := make(map[string]struct{}, len(page.Items)) + for _, m := range page.Items { + ids = append(ids, m.ID) + unique[m.ID] = struct{}{} + } + if !sort.StringsAreSorted(ids) { + t.Fatal("history is not ordered by id") + } + if len(unique) != clients*perClient { + t.Fatalf("%d unique ids, want %d", len(unique), clients*perClient) + } +} + +func TestUnknownRouteAndMethodAreContractErrors(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodGet, "/api/v1/nothing", "", nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodDelete, "/api/v1/users", "", nil, nil), http.StatusMethodNotAllowed, "invalid_request") + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", "{}", map[string]string{"Content-Type": "text/plain"}), http.StatusUnsupportedMediaType, "unsupported_media_type") + big := map[string]string{"username": "big", "display_name": strings.Repeat("x", 70*1024)} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", big, nil), http.StatusRequestEntityTooLarge, "payload_too_large") +} diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go new file mode 100644 index 00000000..d0d4819b --- /dev/null +++ b/internal/app/auth_test.go @@ -0,0 +1,176 @@ +package app_test + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "messenger/internal/user" +) + +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +func TestLoginReturnsBearerSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + before := time.Now() + + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)) + ts.expect(res, http.StatusCreated, "") + s := decode[sessionBody](t, res) + + if s.TokenType != "Bearer" || len(s.Token) != 43 || s.Token == alice.token { + t.Fatalf("session = %+v; want a fresh 43-character Bearer token", s) + } + if s.User != alice.Response { + t.Fatalf("user = %+v, want %+v", s.User, alice.Response) + } + expires, err := time.Parse(time.RFC3339, s.ExpiresAt) + if err != nil || !strings.HasSuffix(s.ExpiresAt, "Z") { + t.Fatalf("expires_at = %q: %v", s.ExpiresAt, err) + } + if ttl := expires.Sub(before); ttl < 59*time.Minute || ttl > 61*time.Minute { + t.Fatalf("expires_at is %v after login, want the 1h TTL", ttl) + } + if strings.Contains(string(res.raw), "password") || strings.Contains(string(res.raw), "argon2") { + t.Fatalf("session response leaks password data: %s", res.raw) + } +} + +func TestLoginFailuresAreUniformBasicChallenges(t *testing.T) { + ts := newTester(t) + ts.user("alice", "Алиса") + encoded := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) } + cases := map[string]map[string]string{ + "wrong password": basic("alice", testPassword+"!"), + "unknown user": basic("mallory", testPassword), + "no header": nil, + "bearer scheme": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "not base64": {"Authorization": "Basic !!!"}, + "no colon": {"Authorization": "Basic " + encoded("alice")}, + "nul username": basic("a\x00b", testPassword), + "non-utf-8 name": basic("\xff\xfe", testPassword), + } + messages := map[string]string{} + for name, headers := range cases { + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != basicChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, basicChallenge) + } + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + messages[name] = eb.Error.Message + } + if messages["wrong password"] != messages["unknown user"] { + t.Fatalf("wrong password and unknown user must look the same: %q vs %q", messages["wrong password"], messages["unknown user"]) + } +} + +// The password is UTF-8 and may contain colons; only the first colon of the +// Basic credentials separates the username (contracts/README.md). +func TestLoginAcceptsColonAndUnicodePasswords(t *testing.T) { + ts := newTester(t) + const password = "пароль:с:двоеточием 👋" + body := map[string]string{"username": "alice", "display_name": "Алиса", "password": password} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") + s := ts.login("alice", password) + if s.User.Username != "alice" { + t.Fatalf("logged in as %+v", s.User) + } +} + +func TestProtectedRoutesAcceptOnlyBearer(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + me := func(headers map[string]string) response { + return ts.do(http.MethodGet, "/api/v1/users/me", "", nil, headers) + } + rejected := map[string]map[string]string{ + "no header": nil, + "basic with valid credentials": basic("alice", testPassword), + "x-user-id alone": {"X-User-Id": alice.ID}, + "unknown token": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "empty token": {"Authorization": "Bearer "}, + "scheme only": {"Authorization": "Bearer"}, + "other scheme": {"Authorization": "Token " + alice.token}, + "two tokens": {"Authorization": "Bearer " + alice.token + " " + eve.token}, + } + for name, headers := range rejected { + res := me(headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != bearerChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, bearerChallenge) + } + } + accepted := map[string]map[string]string{ + "lowercase scheme": {"Authorization": "bearer " + alice.token}, + "several spaces": {"Authorization": "Bearer " + alice.token}, + "x-user-id is ignored": {"Authorization": "Bearer " + alice.token, "X-User-Id": eve.ID}, + } + for name, headers := range accepted { + res := me(headers) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got.ID != alice.ID { + t.Errorf("%s: caller = %s, want alice", name, got.ID) + } + } +} + +func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + other := ts.login("alice", testPassword) + + res := ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil) + ts.expect(res, http.StatusNoContent, "") + if len(res.raw) != 0 { + t.Fatalf("204 must have no body, got %q", res.raw) + } + + for _, again := range []response{ + ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), + ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), + } { + ts.expect(again, http.StatusUnauthorized, "unauthorized") + if again.challenge != bearerChallenge { + t.Fatalf("revoked token: WWW-Authenticate = %q, want %q", again.challenge, bearerChallenge) + } + } + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") +} + +// Valkey enforces the expiry by the Docker VM clock, which may lag this host +// by a fraction of a second, so the 401 is awaited with a tolerance. +func TestSessionExpiresAfterTTL(t *testing.T) { + const ttl, clockTolerance = 2 * time.Second, 3 * time.Second + ts := newTesterWithTTL(t, ttl) + alice := ts.user("alice", "A") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") + time.Sleep(ttl) + deadline := time.Now().Add(clockTolerance) + for { + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + if res.status == http.StatusUnauthorized { + ts.expect(res, http.StatusUnauthorized, "unauthorized") + return + } + if time.Now().After(deadline) { + t.Fatalf("session still valid %v after its TTL of %v", clockTolerance, ttl) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) { + ts := newTester(t) + body := map[string]string{"username": "alice", "display_name": "A", "password": testPassword} + headers := map[string]string{"Authorization": "Bearer garbage"} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, headers), http.StatusCreated, "") +} diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go new file mode 100644 index 00000000..155c8000 --- /dev/null +++ b/internal/app/failure_test.go @@ -0,0 +1,101 @@ +package app_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +// newIsolatedTester assembles the application over containers of its own, +// so a test can stop or pause them without disturbing the shared ones. +func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *testenv.DB, *testenv.KV) { + t.Helper() + ctx := context.Background() + pg, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pg.Close) + vk, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(vk.Close) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{ + Log: log, + DB: pg.Pool, + Sessions: vk.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + RequestTimeout: requestTimeout, + InstanceID: testInstance, + }) + return &tester{t: t, a: a}, pg, vk +} + +// Without the session store no caller can be authenticated: the API answers +// 503, never 401 (the token is not at fault) and never as an anonymous user. +// Registration does not need the store and keeps working. +func TestValkeyOutageAnswers503(t *testing.T) { + ts, _, vk := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + room := ts.chat(alice.token, "t", alice.ID) + + if err := vk.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+room.ID, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") + + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") +} + +func TestPostgresOutageAnswers503(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + + if err := pg.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/chats", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") +} + +// A database that accepts connections but never answers must not hold the +// request: the request deadline turns it into a 503. +func TestHungPostgresAnswers503WithinTheRequestTimeout(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 300*time.Millisecond) + alice := ts.user("alice", "Алиса") + + ctx := context.Background() + if err := pg.Pause(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.Unpause(ctx) }) + + for _, path := range []string{"/api/v1/chats", "/health/ready"} { + start := time.Now() + ts.expect(ts.do(http.MethodGet, path, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("%s took %v with a hung database, want it bounded", path, elapsed) + } + } +} diff --git a/internal/app/idempotency_test.go b/internal/app/idempotency_test.go new file mode 100644 index 00000000..e213baab --- /dev/null +++ b/internal/app/idempotency_test.go @@ -0,0 +1,153 @@ +package app_test + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "messenger/internal/app" +) + +func (ts *tester) send(token, chatID, key string, body any) response { + ts.t.Helper() + return ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, body, map[string]string{"Idempotency-Key": key}) +} + +func TestSendIdempotencyKeyValidation(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "ключи", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + body := map[string]string{"text": "x"} + + ts.expect(ts.do(http.MethodPost, path, alice.token, body, nil), http.StatusBadRequest, "invalid_request") + for name, key := range map[string]string{ + "empty": "", + "too long": strings.Repeat("a", 129), + "inner space": "a b", + "non-ascii": "ключ", + "slash": "a/b", + } { + res := ts.send(alice.token, room.ID, key, body) + if res.status != http.StatusBadRequest { + t.Errorf("%s key: status %d, want 400", name, res.status) + } + } + ts.expect(ts.send(alice.token, room.ID, strings.Repeat("a", 128), body), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "Az09._:-", body), http.StatusCreated, "") + + // Two Idempotency-Key headers are ambiguous: which one names the attempt? + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"x"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Add("Idempotency-Key", "one") + req.Header.Add("Idempotency-Key", "two") + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("two keys: status %d, want 400", rec.Code) + } +} + +func TestReplayReturnsIdenticalBody(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "повторы", alice.ID) + first := ts.send(alice.token, room.ID, "retry-1", map[string]string{"text": "Привет 👋"}) + ts.expect(first, http.StatusCreated, "") + // The same JSON with other spacing is the same request. + again := ts.send(alice.token, room.ID, "retry-1", ` { "text" : "Привет 👋" } `) + ts.expect(again, http.StatusCreated, "") + if !bytes.Equal(first.raw, again.raw) { + t.Fatalf("replay body differs:\n%s\n%s", first.raw, again.raw) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1", len(page.Items)) + } +} + +func TestReplayWithDifferentBodyIs409(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "конфликт", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "первое"}), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "второе"}), http.StatusConflict, "conflict") +} + +// A replay is a new request: a revoked session cannot fetch the stored result. +func TestReplayRechecksSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "сессия", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusCreated, "") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusNoContent, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") +} + +// The header is checked before access, like the JSON syntax: the answer does +// not depend on the chat, so it reveals nothing about a foreign one. +func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + room := ts.chat(alice.token, "чужой", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + ts.expect(ts.do(http.MethodPost, path, eve.token, map[string]string{"text": "x"}, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found") +} + +// Two applications over the same stores stand for two replicas: retries of one +// key alternate between them and still create one message. +func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "реплики", alice.ID) + other := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "00000000000000b2", + }) + engines := []http.Handler{ts.a.Engine, other.Engine} + + const attempts = 10 + type result struct { + status int + body string + instance string + } + results := make([]result, attempts) + var wg sync.WaitGroup + for i := range attempts { + wg.Go(func() { + req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", strings.NewReader(`{"text":"один раз"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Set("Idempotency-Key", "shared") + rec := httptest.NewRecorder() + engines[i%2].ServeHTTP(rec, req) + results[i] = result{rec.Code, rec.Body.String(), rec.Header().Get("X-Instance-Id")} + }) + } + wg.Wait() + instances := map[string]bool{} + for i, r := range results { + if r.status != http.StatusCreated || r.body != results[0].body { + t.Fatalf("attempt %d: %d %s; want 201 %s", i, r.status, r.body, results[0].body) + } + instances[r.instance] = true + } + if len(instances) != 2 { + t.Fatalf("instances %v, want both replicas", instances) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1: %+v", len(page.Items), page.Items) + } +} diff --git a/internal/auth/handler.go b/internal/auth/handler.go new file mode 100644 index 00000000..e8c8ccef --- /dev/null +++ b/internal/auth/handler.go @@ -0,0 +1,89 @@ +package auth + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/user" +) + +// Challenges sent with every 401: Basic on the login endpoint, Bearer on +// every protected one. +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +var errCredentialsRequired = apperr.Unauthorized("basic credentials are required") + +type Handler struct { + sessions *Service +} + +func NewHandler(sessions *Service) *Handler { + return &Handler{sessions: sessions} +} + +// Register mounts login on the public router and logout on the router +// guarded by RequireSession. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/auth/sessions", h.login) + protected.DELETE("/auth/sessions/current", h.logout) +} + +type sessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +// login accepts only the Authorization header; the body is not read. +// BasicAuth splits at the first colon, so a password may contain colons. +func (h *Handler) login(c *gin.Context) { + username, password, ok := c.Request.BasicAuth() + if !ok { + unauthorized(c, basicChallenge, errCredentialsRequired) + return + } + issued, err := h.sessions.Login(c.Request.Context(), username, password) + if err != nil { + unauthorized(c, basicChallenge, err) + return + } + httpx.JSON(c, http.StatusCreated, sessionResponse{ + Token: issued.Token, + TokenType: "Bearer", + ExpiresAt: httpx.Timestamp(issued.ExpiresAt), + User: user.ToResponse(issued.User), + }) +} + +func (h *Handler) logout(c *gin.Context) { + token, ok := c.Get(keyToken) + if !ok { + httpx.Error(c, errors.New("logout route is missing the RequireSession middleware")) + return + } + if err := h.sessions.Logout(c.Request.Context(), token.(string)); err != nil { + httpx.Error(c, err) + return + } + // Status alone is buffered by gin; writing the header now sends the 204 + // with an empty body. + c.Status(http.StatusNoContent) + c.Writer.WriteHeaderNow() +} + +// unauthorized adds the challenge only to a 401: an unavailable store is a +// 503 and must not invite the client to retry its credentials. +func unauthorized(c *gin.Context, challenge string, err error) { + if apperr.KindOf(err) == apperr.KindUnauthorized { + c.Header("WWW-Authenticate", challenge) + } + httpx.Error(c, err) +} diff --git a/internal/auth/main_test.go b/internal/auth/main_test.go new file mode 100644 index 00000000..a542ee95 --- /dev/null +++ b/internal/auth/main_test.go @@ -0,0 +1,33 @@ +package auth_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go new file mode 100644 index 00000000..13dbeeae --- /dev/null +++ b/internal/auth/middleware.go @@ -0,0 +1,48 @@ +package auth + +import ( + "strings" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/httpx" +) + +const keyToken = "messenger.session_token" + +// RequireSession admits a request only with Authorization: Bearer of +// a live session and records its owner for the handlers behind it. Any other +// identification, including Basic credentials and X-User-Id, is ignored, so +// it can neither grant access nor change the caller. +func RequireSession(sessions *Service) gin.HandlerFunc { + return func(c *gin.Context) { + token, ok := bearerToken(c.GetHeader("Authorization")) + if !ok { + unauthorized(c, bearerChallenge, ErrUnauthenticated) + return + } + userID, err := sessions.Authenticate(c.Request.Context(), token) + if err != nil { + unauthorized(c, bearerChallenge, err) + return + } + httpx.SetUserID(c, userID) + c.Set(keyToken, token) + c.Next() + } +} + +// bearerToken parses "Bearer ": the scheme is case-insensitive and +// may be followed by several spaces (RFC 7235); the token itself never +// contains whitespace. +func bearerToken(header string) (string, bool) { + scheme, rest, found := strings.Cut(header, " ") + if !found || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token := strings.TrimLeft(rest, " ") + if token == "" || strings.ContainsAny(token, " \t") { + return "", false + } + return token, true +} diff --git a/internal/auth/repository.go b/internal/auth/repository.go new file mode 100644 index 00000000..4a168f59 --- /dev/null +++ b/internal/auth/repository.go @@ -0,0 +1,18 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores the session until its absolute ExpiresAt. + Create(ctx context.Context, s Session) error + // UserID returns the owner of a live session, or ErrUnauthenticated when + // the token is unknown, expired or revoked: the store cannot tell these + // apart, and the client must not either. + UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) + // Delete revokes the session; deleting a missing one is not an error. + Delete(ctx context.Context, tokenHash string) error +} diff --git a/internal/auth/service.go b/internal/auth/service.go new file mode 100644 index 00000000..458a4744 --- /dev/null +++ b/internal/auth/service.go @@ -0,0 +1,64 @@ +package auth + +import ( + "context" + "time" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/user" +) + +var ErrUnauthenticated = apperr.Unauthorized("a valid bearer token is required") + +// Credentials is what this module needs from the users module: the user +// whose password matches, or user.ErrInvalidCredentials. +type Credentials interface { + Authenticate(ctx context.Context, username, password string) (user.User, error) +} + +// Issued is a new session as the client sees it; Token is shown only here. +type Issued struct { + Token string + ExpiresAt time.Time + User user.User +} + +type Service struct { + users Credentials + sessions Repository + ttl time.Duration +} + +func NewService(users Credentials, sessions Repository, ttl time.Duration) *Service { + return &Service{users: users, sessions: sessions, ttl: ttl} +} + +// Login checks the credentials first, so a store outage never hands out a +// token without a stored session. The expiry is fixed here and never +// extended; it is truncated to milliseconds, the precision of both the API +// timestamps and the store. +func (s *Service) Login(ctx context.Context, username, password string) (Issued, error) { + u, err := s.users.Authenticate(ctx, username, password) + if err != nil { + return Issued{}, err + } + token := newToken() + expiresAt := time.Now().Add(s.ttl).Truncate(time.Millisecond).UTC() + if err := s.sessions.Create(ctx, Session{TokenHash: HashToken(token), UserID: u.ID, ExpiresAt: expiresAt}); err != nil { + return Issued{}, err + } + return Issued{Token: token, ExpiresAt: expiresAt, User: u}, nil +} + +// Authenticate returns the owner of the token's live session or +// ErrUnauthenticated. A store failure stays an unavailability, never +// ErrUnauthenticated: an outage must not look like a bad token to the client. +func (s *Service) Authenticate(ctx context.Context, token string) (uuid.UUID, error) { + return s.sessions.UserID(ctx, HashToken(token)) +} + +func (s *Service) Logout(ctx context.Context, token string) error { + return s.sessions.Delete(ctx, HashToken(token)) +} diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go new file mode 100644 index 00000000..e3cd3fb2 --- /dev/null +++ b/internal/auth/service_test.go @@ -0,0 +1,156 @@ +package auth_test + +import ( + "context" + "encoding/base64" + "errors" + "strings" + "testing" + "time" + + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/platform/apperr" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +type fixture struct { + ctx context.Context + svc *auth.Service + alice user.User +} + +// newFixture wires the real users service and session store over empty +// databases and registers alice. +func newFixture(t *testing.T, ttl time.Duration) fixture { + t.Helper() + db.Reset(t) + kv.Reset(t) + ctx := context.Background() + users := user.NewService(userstore.NewUsers(db.Pool)) + alice, err := users.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatal(err) + } + return fixture{ctx: ctx, svc: auth.NewService(users, authstore.NewSessions(kv.Client), ttl), alice: alice} +} + +func TestLoginIssuesOpaqueTokenBackedByStoredSession(t *testing.T) { + f := newFixture(t, time.Hour) + before := time.Now() + issued, err := f.svc.Login(f.ctx, "alice", pw) + after := time.Now() + if err != nil { + t.Fatalf("Login: %v", err) + } + raw, err := base64.RawURLEncoding.DecodeString(issued.Token) + if err != nil || len(raw) != 32 { + t.Fatalf("token %q must be 32 random bytes in base64url, got %d bytes, %v", issued.Token, len(raw), err) + } + if strings.Contains(issued.Token, f.alice.ID.String()) || strings.Contains(issued.Token, "alice") { + t.Fatal("token must not carry the identity") + } + if issued.User != f.alice { + t.Fatalf("User = %+v, want alice", issued.User) + } + if issued.ExpiresAt.Before(before.Add(time.Hour).Add(-time.Millisecond)) || issued.ExpiresAt.After(after.Add(time.Hour)) { + t.Fatalf("ExpiresAt = %v, want login time + 1h", issued.ExpiresAt) + } + + key := "session:" + auth.HashToken(issued.Token) + owner, err := kv.Client.Get(f.ctx, key).Result() + if err != nil || owner != f.alice.ID.String() { + t.Fatalf("stored owner = %q, %v; want alice", owner, err) + } + expireAt, err := kv.Client.PExpireTime(f.ctx, key).Result() + if err != nil || expireAt != time.Duration(issued.ExpiresAt.UnixMilli())*time.Millisecond { + t.Fatalf("store expiry = %v, %v; want the absolute ExpiresAt %d ms", expireAt, err, issued.ExpiresAt.UnixMilli()) + } + keys, err := kv.Client.Keys(f.ctx, "*").Result() + if err != nil { + t.Fatal(err) + } + for _, k := range keys { + if strings.Contains(k, issued.Token) { + t.Fatal("the raw token must never be stored") + } + } +} + +func TestLoginWithBadCredentialsCreatesNoSession(t *testing.T) { + f := newFixture(t, time.Hour) + for name, in := range map[string][2]string{ + "wrong password": {"alice", pw + "!"}, + "unknown user": {"mallory", pw}, + } { + if _, err := f.svc.Login(f.ctx, in[0], in[1]); !errors.Is(err, user.ErrInvalidCredentials) { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if n, err := kv.Client.DBSize(f.ctx).Result(); err != nil || n != 0 { + t.Fatalf("sessions = %d, %v; want none", n, err) + } +} + +func TestAuthenticateResolvesTheOwner(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if first.Token == second.Token { + t.Fatal("every login must get its own token") + } + for _, token := range []string{first.Token, second.Token} { + id, err := f.svc.Authenticate(f.ctx, token) + if err != nil || id != f.alice.ID { + t.Fatalf("Authenticate = %v, %v; want alice", id, err) + } + } + _, err := f.svc.Authenticate(f.ctx, "unknown-token") + if !errors.Is(err, auth.ErrUnauthenticated) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Fatalf("unknown token: err = %v, want ErrUnauthenticated", err) + } +} + +func TestLogoutRevokesOnlyThatSession(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if err := f.svc.Logout(f.ctx, first.Token); err != nil { + t.Fatalf("Logout: %v", err) + } + if _, err := f.svc.Authenticate(f.ctx, first.Token); !errors.Is(err, auth.ErrUnauthenticated) { + t.Fatalf("revoked token: err = %v, want ErrUnauthenticated", err) + } + if id, err := f.svc.Authenticate(f.ctx, second.Token); err != nil || id != f.alice.ID { + t.Fatalf("other session: %v, %v; want still valid", id, err) + } +} + +// Valkey enforces the expiry by the Docker VM clock, which may lag this host +// by a fraction of a second, so the expiry is awaited with a tolerance. +func TestSessionExpiresAfterTTL(t *testing.T) { + const clockTolerance = 3 * time.Second + f := newFixture(t, time.Second) + issued, err := f.svc.Login(f.ctx, "alice", pw) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil { + t.Fatalf("fresh session: %v", err) + } + time.Sleep(time.Until(issued.ExpiresAt)) + deadline := time.Now().Add(clockTolerance) + for { + _, err := f.svc.Authenticate(f.ctx, issued.Token) + if errors.Is(err, auth.ErrUnauthenticated) { + return + } + if time.Now().After(deadline) { + t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance) + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/internal/auth/session.go b/internal/auth/session.go new file mode 100644 index 00000000..daedb116 --- /dev/null +++ b/internal/auth/session.go @@ -0,0 +1,39 @@ +// Package auth authenticates callers. +// +// A login with Basic credentials creates a server-side session and returns an +// opaque bearer token; every protected request presents the token, and logout +// deletes the session, which revokes the token at once. The store keeps only +// the SHA-256 of a token: a token is a random 256-bit value, so a fast digest +// cannot be reversed by guessing, and a leaked store yields no usable tokens. +// Passwords need the opposite treatment (a slow KDF), see platform/password. +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "time" + + "github.com/google/uuid" +) + +type Session struct { + TokenHash string + UserID uuid.UUID + ExpiresAt time.Time +} + +// newToken returns 32 random bytes in base64url: 43 characters, 256 bits. +func newToken() string { + raw := make([]byte, 32) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(raw) + return base64.RawURLEncoding.EncodeToString(raw) +} + +// HashToken is the key under which a token's session is stored. +func HashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/auth/store/valkey.go b/internal/auth/store/valkey.go new file mode 100644 index 00000000..ad7a36e1 --- /dev/null +++ b/internal/auth/store/valkey.go @@ -0,0 +1,64 @@ +// Package store holds the session store implementations. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "messenger/internal/auth" + "messenger/internal/platform/apperr" +) + +const keyPrefix = "session:" + +// Sessions keeps one key per session, session:, holding the +// owner's id. Valkey expires the key itself, so no cleanup job is needed. +type Sessions struct { + client *redis.Client +} + +var _ auth.Repository = (*Sessions)(nil) + +func NewSessions(client *redis.Client) *Sessions { + return &Sessions{client: client} +} + +// Create sets an absolute expiry (EXAT/PXAT): a restart of the store replays +// it from the append-only file and does not extend the session. +func (r *Sessions) Create(ctx context.Context, s auth.Session) error { + err := r.client.SetArgs(ctx, keyPrefix+s.TokenHash, s.UserID.String(), redis.SetArgs{ExpireAt: s.ExpiresAt}).Err() + if err != nil { + return unavailable(err) + } + return nil +} + +func (r *Sessions) UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) { + raw, err := r.client.Get(ctx, keyPrefix+tokenHash).Result() + if errors.Is(err, redis.Nil) { + return uuid.Nil, auth.ErrUnauthenticated + } + if err != nil { + return uuid.Nil, unavailable(err) + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, fmt.Errorf("session holds a malformed user id: %w", err) + } + return id, nil +} + +func (r *Sessions) Delete(ctx context.Context, tokenHash string) error { + if err := r.client.Del(ctx, keyPrefix+tokenHash).Err(); err != nil { + return unavailable(err) + } + return nil +} + +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "session store unavailable", err) +} diff --git a/internal/chat/chat.go b/internal/chat/chat.go new file mode 100644 index 00000000..e84ff665 --- /dev/null +++ b/internal/chat/chat.go @@ -0,0 +1,86 @@ +// Package chat is the chats module. Entities mirror table rows and +// carry no behaviour; membership is a query rather than a field, and +// ChatWithMembers is the projection the API renders. Both entities derive +// CreatedAt from their UUIDv7 id, so the history order (created_at, id) is +// plain id order. +package chat + +import ( + "bytes" + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxTitleCodePoints = 100 + MaxMembers = 100 +) + +// ErrNotFound covers both a missing chat and a chat the caller is not a +// member of, so outsiders cannot probe for existence. +var ErrNotFound = apperr.NotFound("chat not found") + +// ErrKeyReused answers a send whose Idempotency-Key is already bound to a +// message with a different text. +var ErrKeyReused = apperr.Conflict("Idempotency-Key was already used with a different message") + +type Chat struct { + ID uuid.UUID + Title string + CreatedAt time.Time +} + +type ChatWithMembers struct { + Chat + MemberIDs []uuid.UUID +} + +func NewChat(id uuid.UUID, title string) (Chat, error) { + if strings.TrimSpace(title) == "" { + return Chat{}, apperr.Invalid("title must not be blank") + } + if utf8.RuneCountInString(title) > MaxTitleCodePoints { + return Chat{}, apperr.Invalid(fmt.Sprintf("title must be at most %d characters", MaxTitleCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(title, 0) { + return Chat{}, apperr.Invalid("title must not contain the NUL character") + } + return Chat{ID: id, Title: title, CreatedAt: ident.TimeOf(id)}, nil +} + +// ValidateMembers checks the member list of a new chat and returns its own +// copy ordered by id: members are a set, and this is the one order in which +// every read returns them. +func ValidateMembers(memberIDs []uuid.UUID, creator uuid.UUID) ([]uuid.UUID, error) { + if len(memberIDs) == 0 { + return nil, apperr.Invalid("member_ids must not be empty") + } + if len(memberIDs) > MaxMembers { + return nil, apperr.Invalid(fmt.Sprintf("member_ids must have at most %d entries", MaxMembers)) + } + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + for _, m := range memberIDs { + if m == uuid.Nil { + return nil, apperr.Invalid("member_ids must not contain the nil uuid") + } + if _, dup := seen[m]; dup { + return nil, apperr.Invalid("member_ids must be unique") + } + seen[m] = struct{}{} + } + if _, ok := seen[creator]; !ok { + return nil, apperr.Invalid("member_ids must include the creator") + } + members := slices.Clone(memberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { return bytes.Compare(a[:], b[:]) }) + return members, nil +} diff --git a/internal/chat/chat_test.go b/internal/chat/chat_test.go new file mode 100644 index 00000000..f63f60e3 --- /dev/null +++ b/internal/chat/chat_test.go @@ -0,0 +1,88 @@ +package chat_test + +import ( + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewChatDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + c, err := chat.NewChat(id, "Проект") + if err != nil { + t.Fatalf("NewChat error: %v", err) + } + if c.ID != id || c.Title != "Проект" || !c.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("unexpected chat %+v", c) + } +} + +func TestNewChatValidatesTitle(t *testing.T) { + // PostgreSQL text cannot hold U+0000, so a title with it is invalid input, + // not a storage failure. + for _, title := range []string{"", " ", strings.Repeat("я", 101), "nul\x00inside"} { + _, err := chat.NewChat(ident.New(), title) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("title %q must be rejected as invalid, got %v", title, err) + } + } + if _, err := chat.NewChat(ident.New(), strings.Repeat("я", 100)); err != nil { + t.Errorf("100 code points must be accepted: %v", err) + } +} + +// Members come back in one canonical order (by id) whatever order the client +// sent, so the created chat and every later read of it agree. +func TestValidateMembersCopiesInCanonicalOrder(t *testing.T) { + alice, bob := ident.New(), ident.New() + input := []uuid.UUID{bob, alice} + members, err := chat.ValidateMembers(input, alice) + if err != nil { + t.Fatalf("ValidateMembers error: %v", err) + } + if len(members) != 2 || members[0] != alice || members[1] != bob { + t.Fatalf("members must be ordered by id, got %v", members) + } + input[1] = ident.New() + if members[0] != alice { + t.Fatal("result must not alias the input slice") + } +} + +func TestValidateMembersRejectsInvalidLists(t *testing.T) { + alice, bob := ident.New(), ident.New() + cases := map[string][]uuid.UUID{ + "empty": {}, + "nil": nil, + "missing creator": {bob}, + "duplicate": {alice, bob, bob}, + "duplicate self": {alice, alice}, + "too many": append([]uuid.UUID{alice}, many(100)...), + "nil uuid member": {alice, uuid.Nil}, + } + for name, members := range cases { + _, err := chat.ValidateMembers(members, alice) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: want invalid, got %v", name, err) + } + if err != nil && !strings.Contains(err.Error(), "member_ids") { + t.Errorf("%s: message must name the field: %v", name, err) + } + } + if _, err := chat.ValidateMembers(append([]uuid.UUID{alice}, many(99)...), alice); err != nil { + t.Errorf("100 members must be accepted: %v", err) + } +} + +func many(n int) []uuid.UUID { + out := make([]uuid.UUID, n) + for i := range out { + out[i] = ident.New() + } + return out +} diff --git a/internal/chat/cleanup.go b/internal/chat/cleanup.go new file mode 100644 index 00000000..13db162e --- /dev/null +++ b/internal/chat/cleanup.go @@ -0,0 +1,30 @@ +package chat + +import ( + "context" + "log/slog" + "time" +) + +// RunKeyCleanup purges expired Idempotency-Keys every period until ctx is done. +func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + deleted, err := svc.PurgeExpiredKeys(ctx) + if err != nil { + if ctx.Err() == nil { + log.Warn("idempotency key cleanup failed", "deleted", deleted, "error", err.Error()) + } + continue + } + if deleted > 0 { + log.Info("idempotency keys purged", "deleted", deleted) + } + } +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go new file mode 100644 index 00000000..c913393e --- /dev/null +++ b/internal/chat/handler.go @@ -0,0 +1,209 @@ +package chat + +import ( + "net/http" + "regexp" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +type Handler struct { + chats *Service +} + +func NewHandler(chats *Service) *Handler { + return &Handler{chats: chats} +} + +// Register mounts the chat endpoints on a router guarded by the authentication middleware. +func (h *Handler) Register(protected gin.IRouter) { + protected.POST("/chats", httpx.WithUser(h.create)) + protected.GET("/chats", httpx.WithUser(h.list)) + protected.GET("/chats/:chat_id", httpx.WithUser(h.get)) + protected.POST("/chats/:chat_id/messages", httpx.WithUser(h.send)) + protected.GET("/chats/:chat_id/messages", httpx.WithUser(h.history)) +} + +type createChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type createMessageRequest struct { + Text string `json:"text"` +} + +type ChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type MessageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` +} + +func toChatResponse(c ChatWithMembers) ChatResponse { + members := make([]string, 0, len(c.MemberIDs)) + for _, m := range c.MemberIDs { + members = append(members, m.String()) + } + return ChatResponse{ + ID: c.ID.String(), + Title: c.Title, + MemberIDs: members, + CreatedAt: httpx.Timestamp(c.CreatedAt), + } +} + +func toMessageResponse(m Message) MessageResponse { + return MessageResponse{ + ID: m.ID.String(), + ChatID: m.ChatID.String(), + SenderID: m.SenderID.String(), + Text: m.Text, + CreatedAt: httpx.Timestamp(m.CreatedAt), + } +} + +func (h *Handler) create(c *gin.Context, actor uuid.UUID) { + var req createChatRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + members := make([]uuid.UUID, 0, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + id, err := ident.Parse(raw) + if err != nil { + httpx.Error(c, apperr.Invalid("member_ids must contain valid UUIDs")) + return + } + members = append(members, id) + } + chat, err := h.chats.Create(c.Request.Context(), actor, req.Title, members) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toChatResponse(chat)) +} + +func (h *Handler) list(c *gin.Context, actor uuid.UUID) { + limit, after, err := listing(c, actor) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.List(c.Request.Context(), actor, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, actor, toChatResponse)) +} + +func (h *Handler) get(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + chat, err := h.chats.Get(c.Request.Context(), actor, chatID) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toChatResponse(chat)) +} + +func (h *Handler) send(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + key, err := idempotencyKey(c) + if err != nil { + httpx.Error(c, err) + return + } + var req createMessageRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text}) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toMessageResponse(message)) +} + +func (h *Handler) history(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + limit, after, err := listing(c, chatID) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.History(c.Request.Context(), actor, chatID, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse)) +} + +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +// idempotencyKey rejects two values of the header as ambiguous. +func idempotencyKey(c *gin.Context) (string, error) { + values := c.Request.Header.Values("Idempotency-Key") + if len(values) != 1 || !keyPattern.MatchString(values[0]) { + return "", apperr.Invalid("Idempotency-Key header is required: one value of 1-128 characters A-Z a-z 0-9 . _ : -") + } + return values[0], nil +} + +func chatParam(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("chat_id must be a UUID") + } + return id, nil +} + +// listing parses limit and cursor; the cursor must belong to scope, and a +// present but empty cursor is rejected like a malformed one. +func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) { + limit, err := pagination.ParseLimit(c.Query("limit")) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, err.Error(), err) + } + token, present := c.GetQuery("cursor") + if !present { + return limit, nil, nil + } + last, err := pagination.DecodeCursor(token, scope) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, "cursor is invalid for this resource", err) + } + return limit, &last, nil +} diff --git a/internal/chat/message.go b/internal/chat/message.go new file mode 100644 index 00000000..06a758bc --- /dev/null +++ b/internal/chat/message.go @@ -0,0 +1,50 @@ +package chat + +import ( + "fmt" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const MaxTextCodePoints = 4000 + +type Message struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +// NewMessage keeps the text verbatim; only blank or oversized text is rejected. +func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { + if strings.TrimSpace(text) == "" { + return Message{}, apperr.Invalid("text must not be blank") + } + if utf8.RuneCountInString(text) > MaxTextCodePoints { + return Message{}, apperr.Invalid(fmt.Sprintf("text must be at most %d characters", MaxTextCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(text, 0) { + return Message{}, apperr.Invalid("text must not contain the NUL character") + } + return Message{ + ID: id, + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: ident.TimeOf(id), + }, nil +} + +// SameContent reports whether two messages say the same thing, whatever their +// ids and times. +func (m Message) SameContent(other Message) bool { + return m.Text == other.Text +} diff --git a/internal/chat/message_test.go b/internal/chat/message_test.go new file mode 100644 index 00000000..42762154 --- /dev/null +++ b/internal/chat/message_test.go @@ -0,0 +1,60 @@ +package chat_test + +import ( + "strings" + "testing" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewMessageDerivesCreatedAt(t *testing.T) { + id, chatID, alice := ident.New(), ident.New(), ident.New() + m, err := chat.NewMessage(id, chatID, alice, "Привет, мир 👋") + if err != nil { + t.Fatalf("NewMessage error: %v", err) + } + if m.ID != id || m.ChatID != chatID || m.SenderID != alice || m.Text != "Привет, мир 👋" { + t.Fatalf("unexpected message %+v", m) + } + if !m.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", m.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewMessageValidatesTextByCodePoints(t *testing.T) { + chatID, alice := ident.New(), ident.New() + if _, err := chat.NewMessage(ident.New(), chatID, alice, strings.Repeat("я", 4000)); err != nil { + t.Errorf("4000 code points must be accepted: %v", err) + } + for _, text := range []string{"", " ", "\n\t ", strings.Repeat("я", 4001), "nul\x00inside"} { + _, err := chat.NewMessage(ident.New(), chatID, alice, text) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("text %q must be rejected as invalid, got %v", text, err) + } + } +} + +func TestNewMessageKeepsSurroundingWhitespace(t *testing.T) { + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " hi ") + if err != nil { + t.Fatal(err) + } + if m.Text != " hi " { + t.Fatalf("text must be stored verbatim, got %q", m.Text) + } +} + +func TestSameContentComparesWhatTheSenderWrote(t *testing.T) { + chatID, alice := ident.New(), ident.New() + first, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + retry, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + other, _ := chat.NewMessage(ident.New(), chatID, alice, "привет!") + if !first.SameContent(retry) { + t.Fatal("messages with the same text must have the same content") + } + if first.SameContent(other) { + t.Fatal("messages with different text must differ") + } +} diff --git a/internal/chat/repository.go b/internal/chat/repository.go new file mode 100644 index 00000000..47d929d7 --- /dev/null +++ b/internal/chat/repository.go @@ -0,0 +1,36 @@ +package chat + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +// ChatRepository stores chats and their membership. Membership is answered +// by queries; the projection with members is assembled by the store in one +// query, never by loading a member list into the entity. Members are always +// returned ordered by id. +type ChatRepository interface { + // Create stores the chat row and its members atomically. + Create(ctx context.Context, chat Chat, memberIDs []uuid.UUID) error + // GetForMember returns the chat with members, or ErrNotFound when the + // chat does not exist or the user is not a member. + GetForMember(ctx context.Context, chatID, userID uuid.UUID) (ChatWithMembers, error) + IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) + // ListByMember returns up to limit chats of the user ordered by + // (created_at, id), strictly after the chat identified by after when set. + ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]ChatWithMembers, error) +} + +type MessageRepository interface { + // AppendOnce stores message under the sender's key unless the key was + // used in this chat within ttl, and returns the message the key is bound + // to: the given one or the earlier one. + AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error) + DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) + // ListByChat returns up to limit messages ordered by (created_at, id), + // strictly after the message identified by after when set. An unknown + // chat yields an empty history. + ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]Message, error) +} diff --git a/internal/chat/service.go b/internal/chat/service.go new file mode 100644 index 00000000..ac71e96f --- /dev/null +++ b/internal/chat/service.go @@ -0,0 +1,139 @@ +package chat + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +const ( + keyPurgeBatch = 1000 + // keyPurgeGrace keeps a key past its TTL, so a key that a request has just + // found valid cannot vanish before the request reads its message back. + keyPurgeGrace = time.Minute +) + +// UserLookup is what this module needs from the users module: one batched +// existence check. It is declared here, on the consumer side, so this +// package never imports the users package and tests substitute a map. +type UserLookup interface { + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} + +type Service struct { + chats ChatRepository + messages MessageRepository + users UserLookup + keyTTL time.Duration +} + +// NewService takes keyTTL, how long an Idempotency-Key stays bound to its +// message. +func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service { + return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL} +} + +func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) { + chat, err := NewChat(ident.New(), title) + if err != nil { + return ChatWithMembers{}, err + } + members, err := ValidateMembers(memberIDs, creator) + if err != nil { + return ChatWithMembers{}, err + } + found, err := s.users.ExistingIDs(ctx, members) + if err != nil { + return ChatWithMembers{}, fmt.Errorf("check members: %w", err) + } + if len(found) != len(members) { + return ChatWithMembers{}, apperr.Invalid("member_ids must reference registered users") + } + if err := s.chats.Create(ctx, chat, members); err != nil { + return ChatWithMembers{}, err + } + return ChatWithMembers{Chat: chat, MemberIDs: members}, nil +} + +func (s *Service) Get(ctx context.Context, actor, chatID uuid.UUID) (ChatWithMembers, error) { + return s.chats.GetForMember(ctx, chatID, actor) +} + +func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[ChatWithMembers], error) { + items, err := s.chats.ListByMember(ctx, actor, after, limit+1) + if err != nil { + return pagination.Page[ChatWithMembers]{}, err + } + return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil +} + +// MessagePayload is one send attempt: the message and its Idempotency-Key. +type MessagePayload struct { + ChatID uuid.UUID + IdempotencyKey string + Text string +} + +// Send checks access and the text on every attempt, a replay included, as the +// contract requires of a repeat. +func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) { + if err := s.requireMember(ctx, actor, p.ChatID); err != nil { + return Message{}, err + } + message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text) + if err != nil { + return Message{}, err + } + stored, err := s.messages.AppendOnce(ctx, message, p.IdempotencyKey, s.keyTTL) + if err != nil { + return Message{}, err + } + if !stored.SameContent(message) { + return Message{}, ErrKeyReused + } + return stored, nil +} + +func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) { + var total int64 + for { + n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL+keyPurgeGrace, keyPurgeBatch) + total += n + if err != nil { + return total, err + } + if n < keyPurgeBatch { + return total, nil + } + } +} + +func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) { + if err := s.requireMember(ctx, actor, chatID); err != nil { + return pagination.Page[Message]{}, err + } + items, err := s.messages.ListByChat(ctx, chatID, after, limit+1) + if err != nil { + return pagination.Page[Message]{}, err + } + return pagination.Cut(items, limit, func(m Message) uuid.UUID { return m.ID }), nil +} + +// requireMember runs before any field validation, so an outsider gets the +// same answer for a foreign chat as for a missing one. +func (s *Service) requireMember(ctx context.Context, actor, chatID uuid.UUID) error { + ok, err := s.chats.IsMember(ctx, chatID, actor) + if err != nil { + return fmt.Errorf("check membership: %w", err) + } + if !ok { + return ErrNotFound + } + return nil +} diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go new file mode 100644 index 00000000..1098708c --- /dev/null +++ b/internal/chat/service_test.go @@ -0,0 +1,391 @@ +package chat_test + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +// countingUsers passes through to the real users service and counts the +// lookups, which proves that member checks are batched. +type countingUsers struct { + *user.Service + calls atomic.Int32 +} + +func (c *countingUsers) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + c.calls.Add(1) + return c.Service.ExistingIDs(ctx, ids) +} + +type fixture struct { + svc *chat.Service + users *countingUsers + alice, bob uuid.UUID + eve uuid.UUID + ctx context.Context +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + ctx := context.Background() + users := &countingUsers{Service: user.NewService(userstore.NewUsers(db.Pool))} + register := func(name string) uuid.UUID { + u, err := users.Register(ctx, name, name, "correct-horse-battery") + if err != nil { + t.Fatal(err) + } + return u.ID + } + return fixture{ + svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users, time.Hour), + users: users, + alice: register("alice"), + bob: register("bob"), + eve: register("eve"), + ctx: ctx, + } +} + +func TestCreateChatChecksMembersInOneCall(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "Проект", []uuid.UUID{f.eve, f.alice, f.bob}) + if err != nil { + t.Fatalf("Create error: %v", err) + } + if c.Title != "Проект" || len(c.MemberIDs) != 3 { + t.Fatalf("unexpected chat %+v", c) + } + if got := f.users.calls.Load(); got != 1 { + t.Fatalf("member existence must be checked with one batch call, got %d", got) + } + got, err := f.svc.Get(f.ctx, f.bob, c.ID) + if err != nil { + t.Fatalf("member Get error: %v", err) + } + if fmt.Sprint(got.MemberIDs) != fmt.Sprint(c.MemberIDs) || !got.CreatedAt.Equal(c.CreatedAt) || got.Title != c.Title { + t.Fatalf("Get = %+v, want what Create returned %+v", got, c) + } +} + +func TestCreateChatRejectsUnknownMember(t *testing.T) { + f := newFixture(t) + _, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, ident.New()}) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("kind = %v, want KindInvalid (%v)", apperr.KindOf(err), err) + } + if !strings.Contains(err.Error(), "member_ids") { + t.Fatalf("message must name the field: %v", err) + } +} + +func TestCreateChatRejectsInvalidInputBeforeAnyLookup(t *testing.T) { + f := newFixture(t) + cases := map[string]struct { + title string + members []uuid.UUID + }{ + "blank title": {" ", []uuid.UUID{f.alice}}, + "missing creator": {"t", []uuid.UUID{f.bob}}, + "duplicate": {"t", []uuid.UUID{f.alice, f.alice}}, + } + for name, tc := range cases { + _, err := f.svc.Create(f.ctx, f.alice, tc.title, tc.members) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: kind = %v, want KindInvalid (%v)", name, apperr.KindOf(err), err) + } + } + if got := f.users.calls.Load(); got != 0 { + t.Fatalf("invalid input must be rejected before any user lookup, got %d calls", got) + } +} + +func TestGetHidesForeignAndUnknownChatsAlike(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, foreign := f.svc.Get(f.ctx, f.eve, c.ID) + _, unknown := f.svc.Get(f.ctx, f.eve, ident.New()) + if !errors.Is(foreign, chat.ErrNotFound) || !errors.Is(unknown, chat.ErrNotFound) { + t.Fatalf("foreign = %v, unknown = %v; want both ErrNotFound", foreign, unknown) + } + if apperr.KindOf(foreign) != apperr.KindNotFound { + t.Fatalf("ErrNotFound must carry KindNotFound, got %v", apperr.KindOf(foreign)) + } +} + +func TestListPagesOnlyOwnChats(t *testing.T) { + f := newFixture(t) + var mine []uuid.UUID + for range 3 { + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + mine = append(mine, c.ID) + if _, err := f.svc.Create(f.ctx, f.eve, "t", []uuid.UUID{f.eve}); err != nil { + t.Fatal(err) + } + } + page, err := f.svc.List(f.ctx, f.alice, nil, 2) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 2 || page.Next == nil || *page.Next != mine[1] { + t.Fatalf("first page = %+v", page) + } + if len(page.Items[0].MemberIDs) != 2 { + t.Fatalf("listed chats must carry their members: %+v", page.Items[0]) + } + rest, err := f.svc.List(f.ctx, f.alice, page.Next, 2) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || rest.Items[0].ID != mine[2] || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + all, err := f.svc.List(f.ctx, f.bob, nil, 100) + if err != nil || len(all.Items) != 3 || all.Next != nil { + t.Fatalf("bob page = %+v, %v", all, err) + } +} + +func TestSendChecksAccessBeforeValidation(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, err = f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: " "}) + if !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err) + } + _, err = f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: " "}) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err)) + } + msg, err := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k3", Text: "Привет 👋"}) + if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" { + t.Fatalf("Send = %+v, %v", msg, err) + } +} + +func TestHistoryPagesAndHidesForeignChat(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + first, _ := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: "1"}) + second, _ := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: "2"}) + + page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0] != first || page.Next == nil || *page.Next != first.ID { + t.Fatalf("first page = %+v", page) + } + rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || rest.Items[0] != second || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider history: err = %v, want ErrNotFound", err) + } + empty, err := f.svc.History(f.ctx, f.alice, c.ID, &second.ID, 10) + if err != nil || empty.Items == nil || len(empty.Items) != 0 || empty.Next != nil { + t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err) + } +} + +func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID { + t.Helper() + c, err := f.svc.Create(f.ctx, members[0], "t", members) + if err != nil { + t.Fatal(err) + } + return c.ID +} + +func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message { + t.Helper() + m, err := f.svc.Send(f.ctx, actor, chat.MessagePayload{ChatID: c, IdempotencyKey: key, Text: text}) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestSendReplaysSameBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "привет") + again := f.send(t, f.alice, c, "k", "привет") + if again.ID != first.ID || !again.CreatedAt.Equal(first.CreatedAt) { + t.Fatalf("replay = %+v, want %+v", again, first) + } +} + +func TestSendConflictsOnDifferentBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "первое") + _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "другое"}) + if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err) + } +} + +func TestKeysAreScopedByUserAndChat(t *testing.T) { + f := newFixture(t) + c1 := f.newChat(t, f.alice, f.bob) + c2 := f.newChat(t, f.alice, f.bob) + a1 := f.send(t, f.alice, c1, "k", "текст") + b1 := f.send(t, f.bob, c1, "k", "текст") + a2 := f.send(t, f.alice, c2, "k", "текст") + if a1.ID == b1.ID || a1.ID == a2.ID || b1.ID == a2.ID { + t.Fatalf("one key shared across users or chats: %v %v %v", a1.ID, b1.ID, a2.ID) + } +} + +// A replay checks access again: an outsider presenting a member's key learns +// nothing and the key stays bound to the member's message. +func TestSendChecksAccessBeforeKey(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "секрет") + if _, err := f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "секрет"}); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err) + } + if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID { + t.Fatalf("key rebound after the outsider: %v, want %v", again.ID, first.ID) + } +} + +func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "текст") + if _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: " "}); apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err)) + } +} + +func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + const expired = 2500 + _, err := db.Pool.Exec(f.ctx, ` + WITH m AS ( + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $1, gen_random_uuid(), $2, 'old', now() FROM generate_series(1, $3) + RETURNING id) + INSERT INTO message_idempotency_keys (user_id, chat_id, key, message_id, created_at) + SELECT $2, $1, id::text, id, now() - interval '2 hours' FROM m`, c, f.alice, expired) + if err != nil { + t.Fatal(err) + } + f.send(t, f.alice, c, "fresh", "новое") + deleted, err := f.svc.PurgeExpiredKeys(f.ctx) + if err != nil || deleted != expired { + t.Fatalf("PurgeExpiredKeys = %d, %v; want %d", deleted, err, expired) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want the fresh one", left, err) + } +} + +// A key the claim has just seen valid must survive until the replay lookup, +// so the cleanup leaves keys alone for a while after they expire. +func TestPurgeExpiredKeysWaitsAGraceAfterTTL(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "just-expired", "1") + f.send(t, f.alice, c, "long-expired", "2") + if _, err := db.Pool.Exec(f.ctx, ` + UPDATE message_idempotency_keys + SET created_at = now() - interval '1 hour' - CASE key WHEN 'just-expired' THEN interval '10 seconds' ELSE interval '10 minutes' END`); err != nil { + t.Fatal(err) + } + if deleted, err := f.svc.PurgeExpiredKeys(f.ctx); err != nil || deleted != 1 { + t.Fatalf("PurgeExpiredKeys = %d, %v; want only the long expired key", deleted, err) + } + var left string + if err := db.Pool.QueryRow(f.ctx, `SELECT key FROM message_idempotency_keys`).Scan(&left); err != nil || left != "just-expired" { + t.Fatalf("left %q, %v; want just-expired", left, err) + } +} + +func TestRunKeyCleanupPurgesAndStops(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "old", "старое") + if _, err := db.Pool.Exec(f.ctx, `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(f.ctx) + stopped := make(chan struct{}) + go func() { + chat.RunKeyCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), f.svc, 50*time.Millisecond) + close(stopped) + }() + deadline := time.Now().Add(2 * time.Second) + for { + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil { + t.Fatal(err) + } + if left == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("expired key was not purged within 2s") + } + time.Sleep(20 * time.Millisecond) + } + cancel() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("RunKeyCleanup did not return after cancel") + } +} diff --git a/internal/chat/store/idempotency_test.go b/internal/chat/store/idempotency_test.go new file mode 100644 index 00000000..2b489947 --- /dev/null +++ b/internal/chat/store/idempotency_test.go @@ -0,0 +1,228 @@ +package store_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" +) + +const keyTTL = time.Hour + +// expire makes every stored key older than keyTTL. +func expire(t *testing.T) { + t.Helper() + if _, err := db.Pool.Exec(context.Background(), + `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } +} + +func (f fixture) appendOnce(t *testing.T, m chat.Message, key string) chat.Message { + t.Helper() + got, err := f.messages.AppendOnce(f.ctx, m, key, keyTTL) + if err != nil { + t.Fatal(err) + } + return got +} + +func (f fixture) count(t *testing.T, chatID uuid.UUID) int { + t.Helper() + got, err := f.messages.ListByChat(f.ctx, chatID, nil, 1000) + if err != nil { + t.Fatal(err) + } + return len(got) +} + +func keyTarget(t *testing.T, key string) uuid.UUID { + t.Helper() + var id uuid.UUID + if err := db.Pool.QueryRow(context.Background(), + `SELECT message_id FROM message_idempotency_keys WHERE key = $1`, key).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +func TestAppendOnceInsertsThenReplays(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, first, "k1"); got.ID != first.ID { + t.Fatalf("first append returned %v, want the new message %v", got.ID, first.ID) + } + retry := f.newMessage(t, c.ID, alice) + got := f.appendOnce(t, retry, "k1") + if got.ID != first.ID || !got.CreatedAt.Equal(first.CreatedAt) || got.Text != first.Text || got.SenderID != alice { + t.Fatalf("replay returned %+v, want the stored %+v", got, first) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} + +func TestAppendOnceTakesOverExpiredKey(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + next := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, next, "k1"); got.ID != next.ID { + t.Fatalf("expired key returned %v, want the new message %v", got.ID, next.ID) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } + if target := keyTarget(t, "k1"); target != next.ID { + t.Fatalf("key points to %v, want %v", target, next.ID) + } +} + +// A cleaner batch holds the expired key row; the request waits for it and, +// once the row is gone, inserts its message under the now free key. +func TestAppendOnceWaitsForCleanerThenInserts(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(f.ctx) + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + + next := f.newMessage(t, c.ID, alice) + type result struct { + m chat.Message + err error + } + done := make(chan result, 1) + go func() { + m, err := f.messages.AppendOnce(f.ctx, next, "k1", keyTTL) + done <- result{m, err} + }() + select { + case r := <-done: + t.Fatalf("AppendOnce did not wait for the locked key: %+v", r) + case <-time.After(300 * time.Millisecond): + } + if _, err := tx.Exec(f.ctx, `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil { + t.Fatal(err) + } + if err := tx.Commit(f.ctx); err != nil { + t.Fatal(err) + } + select { + case r := <-done: + if r.err != nil || r.m.ID != next.ID { + t.Fatalf("after the cleaner: %+v, %v; want the new message %v", r.m, r.err, next.ID) + } + case <-time.After(5 * time.Second): + t.Fatal("AppendOnce did not finish after the cleaner committed") + } +} + +// A request holding a key row must not make the cleaner wait: it skips the +// row and deletes it on a later run. +func TestCleanerSkipsRowLockedByRequest(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + start := time.Now() + n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000) + if err != nil || n != 0 { + t.Fatalf("cleaner over a locked row: deleted %d, %v; want 0", n, err) + } + if waited := time.Since(start); waited > time.Second { + t.Fatalf("cleaner waited %v for the locked row", waited) + } + if err := tx.Rollback(f.ctx); err != nil { + t.Fatal(err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000); err != nil || n != 1 { + t.Fatalf("cleaner after release: deleted %d, %v; want 1", n, err) + } +} + +func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + for _, k := range []string{"a", "b", "c"} { + f.appendOnce(t, f.newMessage(t, c.ID, alice), k) + } + expire(t) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "fresh") + + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 2 { + t.Fatalf("first batch deleted %d, %v; want 2", n, err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 1 { + t.Fatalf("second batch deleted %d, %v; want 1", n, err) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want only the fresh one", left, err) + } + if n := f.count(t, c.ID); n != 4 { + t.Fatalf("cleanup removed messages: %d left, want 4", n) + } +} + +// Every request brings its own new message under one key; exactly one is +// stored and all of them get it back. This also proves the key row and the +// message it references are written by one statement. +func TestConcurrentAppendOnceSameKeyStoresOneMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const requests = 20 + ids := make(chan uuid.UUID, requests) + var wg sync.WaitGroup + for range requests { + m := f.newMessage(t, c.ID, alice) + wg.Go(func() { + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + ids <- got.ID + }) + } + wg.Wait() + close(ids) + seen := map[uuid.UUID]int{} + for id := range ids { + seen[id]++ + } + if len(seen) != 1 { + t.Fatalf("requests got %d different messages, want 1: %v", len(seen), seen) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go new file mode 100644 index 00000000..6554bf9f --- /dev/null +++ b/internal/chat/store/postgres.go @@ -0,0 +1,205 @@ +// Package store holds the repository implementations of the chats module. +// +// Keyset pages use `id > COALESCE($after, nil uuid)`: every UUIDv7 is +// greater than the nil UUID, so one query serves the first page and the +// following ones, and the condition stays a range the index can scan. Ids +// are UUIDv7 and created_at derives from them, so ordering by id is the +// contract order (created_at, id). Errors pass through postgres.Classify, so +// an unreachable database surfaces as unavailability. +package store + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/chat" + "messenger/internal/platform/postgres" +) + +type Chats struct { + pool *pgxpool.Pool +} + +var _ chat.ChatRepository = (*Chats)(nil) + +func NewChats(pool *pgxpool.Pool) *Chats { + return &Chats{pool: pool} +} + +func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, + `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`, + c.ID, c.Title, c.CreatedAt); err != nil { + return fmt.Errorf("insert chat: %w", err) + } + if _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id) SELECT $1, unnest($2::uuid[])`, + c.ID, memberIDs); err != nil { + return fmt.Errorf("insert chat members: %w", err) + } + return nil + })) +} + +func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (chat.ChatWithMembers, error) { + row := r.pool.QueryRow(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chats c + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID) + c, err := scanChat(row) + if errors.Is(err, pgx.ErrNoRows) { + return chat.ChatWithMembers{}, chat.ErrNotFound + } + if err != nil { + return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", postgres.Classify(err)) + } + return c, nil +} + +func (r *Chats) IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) { + var ok bool + err := r.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + if err != nil { + return false, fmt.Errorf("check membership: %w", postgres.Classify(err)) + } + return ok, nil +} + +func (r *Chats) ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]chat.ChatWithMembers, error) { + rows, err := r.pool.Query(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chat_members own + JOIN chats c ON c.id = own.chat_id + WHERE own.user_id = $1 + AND own.chat_id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY own.chat_id + LIMIT $3`, + userID, after, limit) + if err != nil { + return nil, fmt.Errorf("select chats: %w", postgres.Classify(err)) + } + chats, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.ChatWithMembers, error) { + return scanChat(row) + }) + if err != nil { + return nil, fmt.Errorf("read chats: %w", postgres.Classify(err)) + } + return chats, nil +} + +func scanChat(row pgx.Row) (chat.ChatWithMembers, error) { + var c chat.ChatWithMembers + if err := row.Scan(&c.ID, &c.Title, &c.CreatedAt, &c.MemberIDs); err != nil { + return chat.ChatWithMembers{}, err + } + return c, nil +} + +type Messages struct { + pool *pgxpool.Pool +} + +var _ chat.MessageRepository = (*Messages)(nil) + +func NewMessages(pool *pgxpool.Pool) *Messages { + return &Messages{pool: pool} +} + +// claimSQL binds the key and inserts the message in one statement, so the +// key's lock never spans a round trip. An expired key is taken over; a valid +// one leaves claimed, and so the message insert, empty. +const claimSQL = ` + WITH claimed AS ( + INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET message_id = EXCLUDED.message_id, created_at = now() + WHERE k.created_at < now() - $5::interval + RETURNING message_id) + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $2, message_id, $1, $6, $7 FROM claimed` + +// The bound message is read by a second statement: the claim's snapshot +// cannot see a key that a concurrent request has just committed. +func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { + tag, err := r.pool.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) + } + if tag.RowsAffected() == 1 { + return m, nil + } + var stored chat.Message + err = r.pool.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) + if err != nil { + return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) + } + return stored, nil +} + +// DeleteExpiredKeys skips rows locked by another cleaner or a request rather +// than waiting for them. +func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) { + tag, err := r.pool.Exec(ctx, ` + DELETE FROM message_idempotency_keys + WHERE (user_id, chat_id, key) IN ( + SELECT user_id, chat_id, key FROM message_idempotency_keys + WHERE created_at < now() - $1::interval + ORDER BY created_at + LIMIT $2 + FOR UPDATE SKIP LOCKED)`, + interval(olderThan), limit) + if err != nil { + return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err)) + } + return tag.RowsAffected(), nil +} + +// interval renders d for a ::interval parameter. +func interval(d time.Duration) string { + return fmt.Sprintf("%d milliseconds", d.Milliseconds()) +} + +// ListByChat is the history query; scripts/explain-history.sql runs the same +// statement for the EXPLAIN evidence. +func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) { + rows, err := r.pool.Query(ctx, ` + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY id + LIMIT $3`, + chatID, after, limit) + if err != nil { + return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) + } + messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { + var m chat.Message + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + return m, err + }) + if err != nil { + return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) + } + return messages, nil +} diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go new file mode 100644 index 00000000..8e843d23 --- /dev/null +++ b/internal/chat/store/postgres_test.go @@ -0,0 +1,320 @@ +package store_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +type fixture struct { + ctx context.Context + chats *store.Chats + messages *store.Messages +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + return fixture{ctx: context.Background(), chats: store.NewChats(db.Pool), messages: store.NewMessages(db.Pool)} +} + +// mustUser stores a user row: chats and messages reference users by FK. +func (f fixture) mustUser(t *testing.T) uuid.UUID { + t.Helper() + id := ident.New() + u, err := user.New(id, "u"+id.String()[24:], "User") + if err != nil { + t.Fatal(err) + } + u.PasswordHash = "x" + if err := userstore.NewUsers(db.Pool).Create(f.ctx, u); err != nil { + t.Fatal(err) + } + return id +} + +func (f fixture) mustChat(t *testing.T, members ...uuid.UUID) chat.Chat { + t.Helper() + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, members); err != nil { + t.Fatal(err) + } + return c +} + +func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), chatID, sender, "text") + if err != nil { + t.Fatal(err) + } + return m +} + +// appendErr stores m under a key of its own, as every send now carries one. +func (f fixture) appendErr(m chat.Message) error { + _, err := f.messages.AppendOnce(f.ctx, m, m.ID.String(), time.Hour) + return err +} + +func TestChatCreateThenGetForMember(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + + got, err := f.chats.GetForMember(f.ctx, c.ID, bob) + if err != nil { + t.Fatal(err) + } + if got.Chat != c || len(got.MemberIDs) != 2 || got.MemberIDs[0] != alice || got.MemberIDs[1] != bob { + t.Fatalf("GetForMember = %+v, want %+v with members [alice bob]", got, c) + } + if _, err := f.chats.GetForMember(f.ctx, c.ID, f.mustUser(t)); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider must get ErrNotFound, got %v", err) + } + for who, want := range map[uuid.UUID]bool{alice: true, bob: true, ident.New(): false} { + ok, err := f.chats.IsMember(f.ctx, c.ID, who) + if err != nil || ok != want { + t.Fatalf("IsMember(%s) = %v, %v; want %v", who, ok, err, want) + } + } +} + +// The chat row and its members are written in one transaction: a failing +// member insert must leave no chat behind. +func TestCreateRollsBackWhenMemberInsertFails(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c, err := chat.NewChat(ident.New(), "half-made") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, ident.New()}); err == nil { + t.Fatal("Create with an unknown member must fail") + } + var chats, members int + if err := db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members)`).Scan(&chats, &members); err != nil { + t.Fatal(err) + } + if chats != 0 || members != 0 { + t.Fatalf("chats = %d, members = %d after a failed create; want 0 and 0", chats, members) + } +} + +func TestChatUnknownID(t *testing.T) { + f := newFixture(t) + if _, err := f.chats.GetForMember(f.ctx, ident.New(), ident.New()); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("GetForMember err = %v, want ErrNotFound", err) + } + if ok, err := f.chats.IsMember(f.ctx, ident.New(), ident.New()); err != nil || ok { + t.Fatalf("IsMember on unknown chat = %v, %v; want false, nil", ok, err) + } +} + +func TestListByMemberFiltersOrdersAndPages(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + var mine []chat.Chat + for range 5 { + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + mine = append(mine, c) + // A chat without alice must never show up in her listing. + f.mustChat(t, bob) + } + // Insert the oldest chat last to prove the store sorts, not appends. + for _, c := range append(mine[1:], mine[0]) { + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, bob}); err != nil { + t.Fatal(err) + } + } + + page, err := f.chats.ListByMember(f.ctx, alice, nil, 3) + if err != nil { + t.Fatal(err) + } + if len(page) != 3 || page[0].ID != mine[0].ID || page[2].ID != mine[2].ID { + t.Fatalf("first page mismatch: %v", page) + } + if len(page[0].MemberIDs) != 2 || page[0].MemberIDs[0] != alice { + t.Fatalf("listing must carry members in id order: %+v", page[0]) + } + last := page[2].ID + rest, err := f.chats.ListByMember(f.ctx, alice, &last, 10) + if err != nil { + t.Fatal(err) + } + if len(rest) != 2 || rest[0].ID != mine[3].ID || rest[1].ID != mine[4].ID { + t.Fatalf("second page mismatch: %v", rest) + } + none, err := f.chats.ListByMember(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if none == nil || len(none) != 0 { + t.Fatalf("unknown member must get an empty non-nil slice, got %#v", none) + } +} + +func TestListByChatUnknownChatIsEmpty(t *testing.T) { + f := newFixture(t) + got, err := f.messages.ListByChat(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if got == nil || len(got) != 0 { + t.Fatalf("want empty non-nil slice, got %#v", got) + } +} + +func TestAppendOutOfOrderIsListedByID(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{third, first, second} { + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0] != first || got[1] != second || got[2] != third { + t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got) + } +} + +func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const total = 250 + var expected []uuid.UUID + for range total { + m := f.newMessage(t, c.ID, alice) + expected = append(expected, m.ID) + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + // Another chat's history must not leak into this one. + other := f.mustChat(t, alice) + if err := f.appendErr(f.newMessage(t, other.ID, alice)); err != nil { + t.Fatal(err) + } + for _, limit := range []int{1, 7, 100} { + var seen []uuid.UUID + var after *uuid.UUID + for { + page, err := f.messages.ListByChat(f.ctx, c.ID, after, limit) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + if len(page) > limit { + t.Fatalf("limit %d: page has %d items", limit, len(page)) + } + for _, m := range page { + seen = append(seen, m.ID) + } + last := page[len(page)-1].ID + after = &last + } + if len(seen) != total { + t.Fatalf("limit %d: walked %d messages, want %d", limit, len(seen), total) + } + for i := range seen { + if seen[i] != expected[i] { + t.Fatalf("limit %d: position %d has %s, want %s", limit, i, seen[i], expected[i]) + } + } + } +} + +func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.newMessage(t, c.ID, alice) + gap := ident.New() // never stored, but sorts between a and b + b := f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{a, b} { + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, &gap, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != b.ID { + t.Fatalf("want only b after the gap id, got %v", got) + } +} + +func TestConcurrentAppendsKeepEveryMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const clients, perClient = 10, 10 + var wg sync.WaitGroup + for range clients { + wg.Go(func() { + for range perClient { + if err := f.appendErr(f.newMessage(t, c.ID, alice)); err != nil { + t.Error(err) + } + } + }) + } + wg.Wait() + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, clients*perClient+1) + if err != nil { + t.Fatal(err) + } + if len(got) != clients*perClient { + t.Fatalf("stored %d messages, want %d", len(got), clients*perClient) + } + for i := 1; i < len(got); i++ { + if bytes.Compare(got[i-1].ID[:], got[i].ID[:]) >= 0 { + t.Fatalf("history out of order at %d", i) + } + } +} diff --git a/internal/health/handler.go b/internal/health/handler.go new file mode 100644 index 00000000..9b14e880 --- /dev/null +++ b/internal/health/handler.go @@ -0,0 +1,77 @@ +// Package health serves the probes. Liveness only proves the process +// answers. Readiness means the process can serve the synchronous API: it is +// switched off during shutdown, so an orchestrator stops routing new traffic +// before the listener closes, and it fails while a critical dependency is +// unreachable. +package health + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +// checkTimeout is shared by all checks of one probe, so the probe answers +// within it even when every dependency hangs; the Docker healthcheck allows +// two seconds. +const checkTimeout = time.Second + +// Check is a dependency the API cannot serve requests without. +type Check struct { + Name string + Probe func(ctx context.Context) error +} + +type Handler struct { + ready atomic.Bool + checks []Check +} + +type status struct { + Status string `json:"status"` +} + +func New(checks ...Check) *Handler { + h := &Handler{checks: checks} + h.ready.Store(true) + return h +} + +func (h *Handler) SetReady(ready bool) { + h.ready.Store(ready) +} + +func (h *Handler) IsReady() bool { + return h.ready.Load() +} + +func (h *Handler) Register(r gin.IRouter) { + r.GET("/health/live", h.live) + r.GET("/health/ready", h.readyz) +} + +func (h *Handler) live(c *gin.Context) { + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} + +func (h *Handler) readyz(c *gin.Context) { + if !h.ready.Load() { + httpx.Error(c, apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(c.Request.Context(), checkTimeout) + defer cancel() + for _, check := range h.checks { + if err := check.Probe(ctx); err != nil { + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, check.Name+" is unavailable", err)) + return + } + } + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} diff --git a/internal/platform/apperr/apperr.go b/internal/platform/apperr/apperr.go new file mode 100644 index 00000000..8de0ff3e --- /dev/null +++ b/internal/platform/apperr/apperr.go @@ -0,0 +1,54 @@ +// Package apperr is the application's error vocabulary: every module reports +// failures as a Kind plus a message that is safe to show to clients, and the +// transport layer owns the only mapping from Kind to HTTP status. Causes are +// kept for logs but never rendered. +package apperr + +import "errors" + +type Kind int + +const ( + KindInternal Kind = iota + KindInvalid + KindUnauthorized + KindNotFound + KindConflict + KindPayloadTooLarge + KindUnsupportedMedia + KindUnavailable +) + +type Error struct { + Kind Kind + Message string + Cause error +} + +func (e *Error) Error() string { return e.Message } + +func (e *Error) Unwrap() error { return e.Cause } + +func New(kind Kind, message string) *Error { + return &Error{Kind: kind, Message: message} +} + +func Wrap(kind Kind, message string, cause error) *Error { + return &Error{Kind: kind, Message: message, Cause: cause} +} + +// KindOf classifies any error; foreign errors are internal. +func KindOf(err error) Kind { + if e, ok := errors.AsType[*Error](err); ok { + return e.Kind + } + return KindInternal +} + +func Invalid(message string) *Error { return New(KindInvalid, message) } +func Unauthorized(message string) *Error { return New(KindUnauthorized, message) } +func NotFound(message string) *Error { return New(KindNotFound, message) } +func Conflict(message string) *Error { return New(KindConflict, message) } +func PayloadTooLarge(message string) *Error { return New(KindPayloadTooLarge, message) } +func UnsupportedMedia(message string) *Error { return New(KindUnsupportedMedia, message) } +func Unavailable(message string) *Error { return New(KindUnavailable, message) } diff --git a/internal/platform/apperr/apperr_test.go b/internal/platform/apperr/apperr_test.go new file mode 100644 index 00000000..6de347f7 --- /dev/null +++ b/internal/platform/apperr/apperr_test.go @@ -0,0 +1,49 @@ +package apperr_test + +import ( + "errors" + "fmt" + "testing" + + "messenger/internal/platform/apperr" +) + +func TestKindOfReturnsKindForApplicationError(t *testing.T) { + err := apperr.NotFound("chat not found") + if got := apperr.KindOf(err); got != apperr.KindNotFound { + t.Fatalf("KindOf = %v, want KindNotFound", got) + } + if err.Error() != "chat not found" { + t.Fatalf("Error() = %q", err.Error()) + } +} + +func TestKindOfSeesThroughWrapping(t *testing.T) { + err := fmt.Errorf("service: %w", apperr.Conflict("username taken")) + if got := apperr.KindOf(err); got != apperr.KindConflict { + t.Fatalf("KindOf = %v, want KindConflict", got) + } +} + +func TestKindOfDefaultsToInternal(t *testing.T) { + if got := apperr.KindOf(errors.New("boom")); got != apperr.KindInternal { + t.Fatalf("KindOf = %v, want KindInternal", got) + } + if got := apperr.KindOf(nil); got != apperr.KindInternal { + t.Fatalf("KindOf(nil) = %v, want KindInternal", got) + } +} + +func TestWrapKeepsCause(t *testing.T) { + cause := errors.New("cause") + err := apperr.Wrap(apperr.KindInvalid, "bad input", cause) + if !errors.Is(err, cause) { + t.Fatal("wrapped error must unwrap to cause") + } + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatal("wrapped error must keep its kind") + } + if err.Error() != "bad input" { + t.Fatalf("Error() = %q, want public message only", err.Error()) + } +} diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go new file mode 100644 index 00000000..936b8ddc --- /dev/null +++ b/internal/platform/config/config.go @@ -0,0 +1,187 @@ +// Package config reads the process configuration from the environment. +// Settings have defaults suited to the lab stand, except credentials, which +// are required and read from files named by *_FILE variables so that their +// values never appear in the environment. An invalid value stops the process +// at start-up with a message naming the variable, never the secret. +package config + +import ( + "fmt" + "log/slog" + "net" + "os" + "strconv" + "strings" + "time" +) + +type Logging struct { + Env string + Level slog.Level +} + +func (l Logging) NewLogger() *slog.Logger { + opts := &slog.HandlerOptions{Level: l.Level} + if l.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} + +type Config struct { + Logging + Host string + Port int + ShutdownTimeout time.Duration + MaxBodyBytes int64 + DatabaseURL string + SessionStoreURL string + SessionTTL time.Duration + // DrainDelay keeps a stopping process serving with readiness off; it must + // exceed the balancer's detection time. + DrainDelay time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DebugAddr string +} + +type Migrate struct { + Logging + DatabaseURL string +} + +// Lookup reads one variable, reporting whether it was set; os.LookupEnv +// in production, a map in tests. +type Lookup func(key string) (string, bool) + +func Load(lookup Lookup) (Config, error) { + cfg := Config{ + Host: "0.0.0.0", + Port: 8080, + ShutdownTimeout: 8 * time.Second, + MaxBodyBytes: 64 * 1024, + SessionTTL: time.Hour, + DrainDelay: 2 * time.Second, + IdempotencyTTL: 24 * time.Hour, + DBMaxConns: 10, + } + if v, ok := lookup("APP_HOST"); ok { + if strings.TrimSpace(v) == "" { + return Config{}, fmt.Errorf("APP_HOST must not be empty") + } + cfg.Host = v + } + if v, ok := lookup("APP_PORT"); ok { + port, err := strconv.Atoi(v) + if err != nil || port < 1 || port > 65535 { + return Config{}, fmt.Errorf("APP_PORT must be an integer between 1 and 65535, got %q", v) + } + cfg.Port = port + } + var err error + if cfg.Logging, err = loadLogging(lookup); err != nil { + return Config{}, err + } + if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return Config{}, fmt.Errorf("APP_SHUTDOWN_TIMEOUT must be a positive duration such as 8s, got %q", v) + } + cfg.ShutdownTimeout = d + } + if v, ok := lookup("APP_MAX_BODY_BYTES"); ok { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n <= 0 { + return Config{}, fmt.Errorf("APP_MAX_BODY_BYTES must be a positive integer, got %q", v) + } + cfg.MaxBodyBytes = n + } + if v, ok := lookup("SESSION_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.SessionTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("APP_DRAIN_DELAY"); ok { + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return Config{}, fmt.Errorf("APP_DRAIN_DELAY must be a non-negative duration such as 2s, got %q", v) + } + cfg.DrainDelay = d + } + if v, ok := lookup("IDEMPOTENCY_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.IdempotencyTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("DB_MAX_CONNS"); ok { + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 || n > 1000 { + return Config{}, fmt.Errorf("DB_MAX_CONNS must be an integer between 1 and 1000, got %q", v) + } + cfg.DBMaxConns = int32(n) + } + if v, ok := lookup("APP_DEBUG_ADDR"); ok && v != "" { + if _, _, err := net.SplitHostPort(v); err != nil { + return Config{}, fmt.Errorf("APP_DEBUG_ADDR must be host:port such as :9090, got %q", v) + } + cfg.DebugAddr = v + } + if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + return Config{}, err + } + if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { + return Config{}, err + } + return cfg, nil +} + +func LoadMigrate(lookup Lookup) (Migrate, error) { + var cfg Migrate + var err error + if cfg.Logging, err = loadLogging(lookup); err != nil { + return Migrate{}, err + } + if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + return Migrate{}, err + } + return cfg, nil +} + +func loadLogging(lookup Lookup) (Logging, error) { + l := Logging{Env: "lab", Level: slog.LevelInfo} + if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { + l.Env = v + } + if v, ok := lookup("APP_LOG_LEVEL"); ok { + if err := l.Level.UnmarshalText([]byte(v)); err != nil { + return Logging{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + } + } + return l, nil +} + +// secret reads the file named by the variable; surrounding whitespace, +// including the trailing newline editors add, is dropped. +func secret(lookup Lookup, key string) (string, error) { + path, ok := lookup(key) + if !ok || strings.TrimSpace(path) == "" { + return "", fmt.Errorf("%s must name a file with the value", key) + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("%s: cannot read %s: %w", key, path, err) + } + value := strings.TrimSpace(string(raw)) + if value == "" { + return "", fmt.Errorf("%s: file %s is empty", key, path) + } + return value, nil +} + +func (c Config) Addr() string { + return net.JoinHostPort(c.Host, strconv.Itoa(c.Port)) +} diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go new file mode 100644 index 00000000..e58ffe19 --- /dev/null +++ b/internal/platform/config/config_test.go @@ -0,0 +1,177 @@ +package config_test + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/platform/config" +) + +func env(values map[string]string) func(string) (string, bool) { + return func(key string) (string, bool) { + v, ok := values[key] + return v, ok + } +} + +func secretFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +const ( + databaseURL = "postgres://app:s3cret@db:5432/messenger" + sessionStoreURL = "redis://:s3cret@valkey:6379/0" +) + +// required holds the variables without defaults, merged with extra. +func required(t *testing.T, extra map[string]string) map[string]string { + t.Helper() + values := map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "SESSION_STORE_URL_FILE": secretFile(t, sessionStoreURL+"\r\n"), + } + for k, v := range extra { + values[k] = v + } + return values +} + +func TestDefaults(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "0.0.0.0:8080" { + t.Fatalf("Addr = %q", cfg.Addr()) + } + if cfg.Env != "lab" || cfg.Level != slog.LevelInfo { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL { + t.Fatalf("urls = %q, %q; want the file contents without line endings", cfg.DatabaseURL, cfg.SessionStoreURL) + } + if cfg.SessionTTL != time.Hour { + t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL) + } + if cfg.DrainDelay != 2*time.Second || cfg.IdempotencyTTL != 24*time.Hour || cfg.DBMaxConns != 10 || cfg.DebugAddr != "" { + t.Fatalf("lab3 defaults = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } +} + +func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { + for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} { + cases := map[string]func(map[string]string){ + "unset": func(v map[string]string) { delete(v, key) }, + "empty path": func(v map[string]string) { v[key] = "" }, + "no file": func(v map[string]string) { v[key] = filepath.Join(t.TempDir(), "missing") }, + "blank file": func(v map[string]string) { v[key] = secretFile(t, " \r\n") }, + } + for name, breakIt := range cases { + values := required(t, nil) + breakIt(values) + _, err := config.Load(env(values)) + if err == nil { + t.Errorf("%s %s: want error", key, name) + continue + } + if !strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "s3cret") { + t.Errorf("%s %s: error must name the variable and hide the secret: %v", key, name, err) + } + } + } +} + +func TestOverrides(t *testing.T) { + cfg, err := config.Load(env(required(t, map[string]string{ + "APP_HOST": "127.0.0.1", + "APP_PORT": "9090", + "APP_ENV": "dev", + "APP_LOG_LEVEL": "debug", + "APP_SHUTDOWN_TIMEOUT": "3s", + "APP_MAX_BODY_BYTES": "1024", + "SESSION_TTL_SECONDS": "300", + "APP_DRAIN_DELAY": "0s", + "IDEMPOTENCY_TTL_SECONDS": "60", + "DB_MAX_CONNS": "25", + "APP_DEBUG_ADDR": ":9090", + }))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.Level != slog.LevelDebug { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.DrainDelay != 0 || cfg.IdempotencyTTL != time.Minute || cfg.DBMaxConns != 25 || cfg.DebugAddr != ":9090" { + t.Fatalf("lab3 overrides = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } +} + +func TestInvalidValuesAreReported(t *testing.T) { + cases := map[string]map[string]string{ + "port text": {"APP_PORT": "abc"}, + "port zero": {"APP_PORT": "0"}, + "port too big": {"APP_PORT": "70000"}, + "level": {"APP_LOG_LEVEL": "loud"}, + "timeout": {"APP_SHUTDOWN_TIMEOUT": "soon"}, + "timeout negative": {"APP_SHUTDOWN_TIMEOUT": "-1s"}, + "body zero": {"APP_MAX_BODY_BYTES": "0"}, + "body text": {"APP_MAX_BODY_BYTES": "big"}, + "empty host": {"APP_HOST": ""}, + "ttl zero": {"SESSION_TTL_SECONDS": "0"}, + "ttl negative": {"SESSION_TTL_SECONDS": "-5"}, + "ttl text": {"SESSION_TTL_SECONDS": "hour"}, + "ttl fraction": {"SESSION_TTL_SECONDS": "1.5"}, + "drain negative": {"APP_DRAIN_DELAY": "-1s"}, + "drain text": {"APP_DRAIN_DELAY": "abc"}, + "key ttl zero": {"IDEMPOTENCY_TTL_SECONDS": "0"}, + "key ttl text": {"IDEMPOTENCY_TTL_SECONDS": "day"}, + "pool zero": {"DB_MAX_CONNS": "0"}, + "pool too big": {"DB_MAX_CONNS": "1001"}, + "pool text": {"DB_MAX_CONNS": "many"}, + "debug addr": {"APP_DEBUG_ADDR": "nonsense"}, + } + for name, values := range cases { + _, err := config.Load(env(required(t, values))) + if err == nil { + t.Errorf("%s: want error", name) + continue + } + for key := range values { + if !strings.Contains(err.Error(), key) { + t.Errorf("%s: error %q does not name %s", name, err, key) + } + } + } +} + +func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) { + cfg, err := config.LoadMigrate(env(map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "APP_ENV": "dev", + "APP_LOG_LEVEL": "warn", + })) + if err != nil { + t.Fatalf("LoadMigrate error: %v", err) + } + if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.Level != slog.LevelWarn { + t.Fatalf("cfg = %+v", cfg) + } + if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") { + t.Fatalf("missing database url: err = %v", err) + } +} diff --git a/internal/platform/httpx/decode.go b/internal/platform/httpx/decode.go new file mode 100644 index 00000000..ce8eeae7 --- /dev/null +++ b/internal/platform/httpx/decode.go @@ -0,0 +1,70 @@ +package httpx + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strings" + "unicode/utf8" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +// DecodeJSON reads exactly one JSON object into dst and enforces what gin's +// binding does not: the media type must be application/json (415), invalid +// UTF-8, unknown fields, wrong types, malformed syntax and trailing data are +// 400, an oversized body is 413, and messages never expose Go type names. +func DecodeJSON(c *gin.Context, dst any) error { + mediaType, _, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || mediaType != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + // The body is already capped by the bodyLimit middleware, so reading it + // whole is bounded. + body, err := io.ReadAll(c.Request.Body) + if err != nil { + return decodeError(err) + } + // encoding/json silently replaces invalid UTF-8 with U+FFFD; the contract + // and RFC 8259 require UTF-8, so such a body is rejected, not repaired. + if !utf8.Valid(body) { + return apperr.Invalid("request body must be valid UTF-8") + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return decodeError(err) + } + if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return apperr.Invalid("unexpected data after JSON body") + } + return nil +} + +func decodeError(err error) error { + if tooLarge, ok := errors.AsType[*http.MaxBytesError](err); ok { + return apperr.Wrap(apperr.KindPayloadTooLarge, fmt.Sprintf("request body must be at most %d bytes", tooLarge.Limit), err) + } + if errors.Is(err, io.EOF) { + return apperr.Wrap(apperr.KindInvalid, "request body is required", err) + } + if wrongType, ok := errors.AsType[*json.UnmarshalTypeError](err); ok { + if wrongType.Field == "" { + return apperr.Wrap(apperr.KindInvalid, "request body must be a JSON object", err) + } + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("field %q has wrong type", wrongType.Field), err) + } + // encoding/json reports unknown fields as a plain error without a type. + const unknownPrefix = "json: unknown field " + if msg := err.Error(); strings.HasPrefix(msg, unknownPrefix) { + field := strings.Trim(strings.TrimPrefix(msg, unknownPrefix), `"`) + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("unknown field %q", field), err) + } + return apperr.Wrap(apperr.KindInvalid, "malformed JSON body", err) +} diff --git a/internal/platform/httpx/engine.go b/internal/platform/httpx/engine.go new file mode 100644 index 00000000..4f1f89ad --- /dev/null +++ b/internal/platform/httpx/engine.go @@ -0,0 +1,50 @@ +// Package httpx is the transport layer over gin: request ids, access +// logging, panic recovery, body limits, strict JSON decoding and the single +// error format of the API contract. gin is used for routing and the +// middleware chain; everything here exists because the contract demands a +// specific behaviour that gin's own helpers do not provide. +package httpx + +import ( + "log/slog" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +const ( + keyRequestID = "messenger.request_id" + keyLogger = "messenger.logger" +) + +// NewEngine builds an engine whose every response, including routing +// failures and panics, carries X-Instance-Id, X-Request-Id and the contract +// error body. +func NewEngine(log *slog.Logger, maxBodyBytes int64, instanceID string) *gin.Engine { + gin.SetMode(gin.ReleaseMode) + e := gin.New() + // Redirects would answer with text/plain and without a request id. + e.RedirectTrailingSlash = false + e.RedirectFixedPath = false + e.HandleMethodNotAllowed = true + + e.Use(instance(instanceID), requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) + e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) }) + e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") }) + return e +} + +func RequestID(c *gin.Context) string { + return c.GetString(keyRequestID) +} + +func logger(c *gin.Context) *slog.Logger { + if l, ok := c.Get(keyLogger); ok { + if log, ok := l.(*slog.Logger); ok { + return log + } + } + return slog.Default() +} diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go new file mode 100644 index 00000000..30c64aca --- /dev/null +++ b/internal/platform/httpx/httpx_test.go @@ -0,0 +1,409 @@ +package httpx_test + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type payload struct { + Text string `json:"text"` + Count int `json:"count"` +} + +const testInstance = "0123456789abcdef" + +func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine { + t.Helper() + return newEngineAt(t, logs, slog.LevelDebug) +} + +func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine { + t.Helper() + if logs == nil { + logs = &bytes.Buffer{} + } + log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level})) + e := httpx.NewEngine(log, 64, testInstance) + e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": ""}) }) + e.GET("/panic", func(c *gin.Context) { panic("secret detail") }) + e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) }) + e.GET("/notfound", func(c *gin.Context) { httpx.Error(c, apperr.NotFound("chat not found")) }) + e.GET("/who", func(c *gin.Context) { + httpx.SetUserID(c, uuid.MustParse("11111111-1111-4111-8111-111111111111")) + id, ok := httpx.UserID(c) + httpx.JSON(c, http.StatusOK, gin.H{"id": id.String(), "ok": ok}) + }) + e.POST("/decode", func(c *gin.Context) { + var p payload + if err := httpx.DecodeJSON(c, &p); err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, p) + }) + return e +} + +func do(t *testing.T, e *gin.Engine, method, path string, body string, headers map[string]string) (*httptest.ResponseRecorder, errorBody) { + t.Helper() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, path, reader) + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + var eb errorBody + if rec.Code >= 400 { + if err := json.Unmarshal(rec.Body.Bytes(), &eb); err != nil { + t.Fatalf("%s %s: error body is not JSON: %v: %s", method, path, err, rec.Body.String()) + } + } + return rec, eb +} + +func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, status int, code string) { + t.Helper() + if rec.Code != status { + t.Fatalf("status = %d, want %d (body %s)", rec.Code, status, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Fatalf("Content-Type = %q", ct) + } + rid := rec.Header().Get("X-Request-Id") + if rid == "" { + t.Fatal("missing X-Request-Id header") + } + if eb.Error.RequestID != rid { + t.Fatalf("body request_id %q != header %q", eb.Error.RequestID, rid) + } + if eb.Error.Code != code || eb.Error.Message == "" { + t.Fatalf("error = %+v, want code %q with a message", eb.Error, code) + } +} + +// The balancer spreads one client's requests over the replicas; the header +// is how a client tells which process answered, so no response may lack it. +func TestEveryResponseCarriesInstanceID(t *testing.T) { + e := newEngine(t, nil) + for _, r := range []struct{ method, path string }{ + {http.MethodGet, "/ok"}, + {http.MethodGet, "/missing"}, + {http.MethodDelete, "/ok"}, + {http.MethodGet, "/panic"}, + {http.MethodGet, "/fail"}, + {http.MethodPost, "/decode"}, + } { + rec, _ := do(t, e, r.method, r.path, "", nil) + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + t.Errorf("%s %s (status %d): X-Instance-Id = %q, want %q", r.method, r.path, rec.Code, got, testInstance) + } + } +} + +func TestUnknownRouteIsJSON404(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestTrailingSlashIsNotRedirected(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/ok/", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestWrongMethodIsJSON405WithAllow(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodDelete, "/ok", "", nil) + assertError(t, rec, eb, http.StatusMethodNotAllowed, "invalid_request") + if !strings.Contains(rec.Header().Get("Allow"), "GET") { + t.Fatalf("Allow = %q, want GET", rec.Header().Get("Allow")) + } +} + +func TestPanicBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/panic", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "secret detail") { + t.Fatal("panic value leaked to the client") + } + if !strings.Contains(logs.String(), "secret detail") { + t.Fatal("panic must be logged for operators") + } +} + +func TestForeignErrorBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/fail", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "password") { + t.Fatal("internal error text leaked to the client") + } + if !strings.Contains(logs.String(), "db password leaked") { + t.Fatal("cause must be logged for operators") + } +} + +func TestApplicationErrorMapping(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/notfound", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") + if eb.Error.Message != "chat not found" { + t.Fatalf("message = %q", eb.Error.Message) + } +} + +func TestSuccessResponseHeadersAndEscaping(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json; charset=utf-8" { + t.Fatalf("Content-Type = %q", ct) + } + if rec.Header().Get("X-Request-Id") == "" { + t.Fatal("missing X-Request-Id on success") + } + if !strings.Contains(rec.Body.String(), `""`) { + t.Fatalf("HTML escaping must be off, body %s", rec.Body.String()) + } +} + +// The balancer sets X-Request-Id on every request, overwriting what the +// client sent; the API keeps it so that one id spans both logs. +func TestRequestIDFromTheBalancerIsKept(t *testing.T) { + const fromLB = "0123456789abcdef0123456789abcdef" + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": fromLB}) + if got := rec.Header().Get("X-Request-Id"); got != fromLB { + t.Fatalf("X-Request-Id = %q, want %q", got, fromLB) + } +} + +func TestMalformedRequestIDIsReplaced(t *testing.T) { + for _, bad := range []string{"has space", "semi;colon", "tab\there", strings.Repeat("a", 65)} { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": bad}) + if got := rec.Header().Get("X-Request-Id"); got == "" || got == bad { + t.Errorf("X-Request-Id for %q = %q, want a generated value", bad, got) + } + } +} + +func TestDecodeJSONHappyPath(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodPost, "/decode", `{"text":"Привет","count":2}`, map[string]string{"Content-Type": "application/json; charset=utf-8"}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"Привет"`) { + t.Fatalf("status %d body %s", rec.Code, rec.Body.String()) + } +} + +func TestDecodeJSONFailures(t *testing.T) { + jsonCT := map[string]string{"Content-Type": "application/json"} + cases := []struct { + name string + body string + headers map[string]string + status int + code string + contain string + }{ + {"no content type", `{"text":"x"}`, nil, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"wrong content type", `{"text":"x"}`, map[string]string{"Content-Type": "text/plain"}, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"empty body", "", jsonCT, http.StatusBadRequest, "invalid_request", "body"}, + {"malformed", `{"text":`, jsonCT, http.StatusBadRequest, "invalid_request", "malformed"}, + {"invalid utf-8", "{\"text\":\"bad \xff byte\"}", jsonCT, http.StatusBadRequest, "invalid_request", "UTF-8"}, + {"unknown field", `{"text":"x","extra":1}`, jsonCT, http.StatusBadRequest, "invalid_request", "extra"}, + {"wrong type", `{"text":123}`, jsonCT, http.StatusBadRequest, "invalid_request", "text"}, + {"trailing data", `{"text":"x"} {"text":"y"}`, jsonCT, http.StatusBadRequest, "invalid_request", "after"}, + {"array instead of object", `[1,2]`, jsonCT, http.StatusBadRequest, "invalid_request", ""}, + {"too large", `{"text":"` + strings.Repeat("x", 100) + `"}`, jsonCT, http.StatusRequestEntityTooLarge, "payload_too_large", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodPost, "/decode", tc.body, tc.headers) + assertError(t, rec, eb, tc.status, tc.code) + if tc.contain != "" && !strings.Contains(eb.Error.Message, tc.contain) { + t.Fatalf("message %q must mention %q", eb.Error.Message, tc.contain) + } + if strings.Contains(eb.Error.Message, "httpx_test") || strings.Contains(eb.Error.Message, "payload") { + t.Fatalf("message must not leak Go type names: %q", eb.Error.Message) + } + }) + } +} + +func TestAccessLogHasRequestFieldsButNoBody(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + rec, _ := do(t, e, http.MethodPost, "/decode", `{"text":"confidential"}`, map[string]string{"Content-Type": "application/json"}) + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + for _, key := range []string{"request_id", "method", "path", "status", "duration_ms"} { + if _, ok := entry[key]; !ok { + t.Errorf("log entry missing %q: %s", key, line) + } + } + if entry["request_id"] != rec.Header().Get("X-Request-Id") { + t.Errorf("log request_id %v != header %q", entry["request_id"], rec.Header().Get("X-Request-Id")) + } + if strings.Contains(logs.String(), "confidential") { + t.Fatal("request body must never be logged") + } +} + +func TestAccessLogLevelFollowsStatus(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngineAt(t, logs, slog.LevelInfo) + + do(t, e, http.MethodGet, "/ok", "", nil) + if strings.Contains(logs.String(), `"msg":"request"`) { + t.Fatalf("successful requests must be logged at debug level only, got %s", logs.String()) + } + do(t, e, http.MethodGet, "/notfound", "", nil) + if entry := logEntry(t, logs); entry["level"] != "INFO" || entry["status"] != float64(404) { + t.Fatalf("client error must be logged at info level with its status, got %v", entry) + } + do(t, e, http.MethodGet, "/fail", "", nil) + if entry := logEntry(t, logs); entry["level"] != "ERROR" || entry["status"] != float64(500) { + t.Fatalf("server error must be logged at error level with its status, got %v", entry) + } + + verbose := &bytes.Buffer{} + do(t, newEngineAt(t, verbose, slog.LevelDebug), http.MethodGet, "/ok", "", nil) + if entry := logEntry(t, verbose); entry["level"] != "DEBUG" || entry["status"] != float64(200) { + t.Fatalf("at debug level successful requests must be logged, got %v", entry) + } +} + +func TestUserIDRoundTripAndAccessLogAttribution(t *testing.T) { + logs := &bytes.Buffer{} + rec, _ := do(t, newEngine(t, logs), http.MethodGet, "/who", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"ok":true`) { + t.Fatalf("UserID must return what SetUserID stored: %d %s", rec.Code, rec.Body.String()) + } + if entry := logEntry(t, logs); entry["user_id"] != "11111111-1111-4111-8111-111111111111" { + t.Fatalf("access log must attribute the request to the user, got %v", entry) + } + anonymous := &bytes.Buffer{} + do(t, newEngine(t, anonymous), http.MethodGet, "/ok", "", nil) + if _, present := logEntry(t, anonymous)["user_id"]; present { + t.Fatal("anonymous requests must not carry user_id") + } +} + +func TestWithUserPassesTheIdentifiedCaller(t *testing.T) { + actor := uuid.MustParse("22222222-2222-4222-8222-222222222222") + e := newEngine(t, nil) + e.GET("/actor", + func(c *gin.Context) { httpx.SetUserID(c, actor) }, + httpx.WithUser(func(c *gin.Context, got uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{"id": got.String()}) + })) + rec, _ := do(t, e, http.MethodGet, "/actor", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), actor.String()) { + t.Fatalf("handler did not receive the caller: %d %s", rec.Code, rec.Body.String()) + } +} + +// Reaching WithUser without an authentication middleware in front is a +// wiring mistake, so it must not be blamed on the client with a 401. +func TestWithUserWithoutIdentityIsAWiringError(t *testing.T) { + e := newEngine(t, nil) + e.GET("/actor", httpx.WithUser(func(c *gin.Context, _ uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{}) + })) + rec, eb := do(t, e, http.MethodGet, "/actor", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") +} + +// A 5xx tells the client only the safe message; the operator must still see +// why the dependency failed, so the cause goes to the log. +func TestUnavailableCauseIsLoggedButNotReturned(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + e.GET("/down", func(c *gin.Context) { + cause := errors.New("dial tcp 10.0.0.7:5432: connect: connection refused") + httpx.Error(c, fmt.Errorf("select user: %w", apperr.Wrap(apperr.KindUnavailable, "database unavailable", cause))) + }) + rec, eb := do(t, e, http.MethodGet, "/down", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + // Neither the cause nor the wrapping context of internal operations + // ("select user: ...") may reach the client. + if eb.Error.Message != "database unavailable" { + t.Fatalf("client message = %q, want only the application message", eb.Error.Message) + } + if !strings.Contains(logs.String(), "connection refused") || !strings.Contains(logs.String(), eb.Error.RequestID) { + t.Fatalf("log must carry the cause and the request id, got:\n%s", logs.String()) + } +} + +// Everything below Timeout sees one deadline, so waiting on a dependency +// ends with the request instead of holding the client. +func TestTimeoutBoundsTheRequestContext(t *testing.T) { + e := newEngine(t, nil) + e.GET("/slow", httpx.Timeout(50*time.Millisecond), func(c *gin.Context) { + select { + case <-c.Request.Context().Done(): + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, "gave up", c.Request.Context().Err())) + case <-time.After(5 * time.Second): + httpx.JSON(c, http.StatusOK, gin.H{}) + } + }) + start := time.Now() + rec, eb := do(t, e, http.MethodGet, "/slow", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("request took %v, want it cut at the 50ms deadline", elapsed) + } +} + +func logEntry(t *testing.T, logs *bytes.Buffer) map[string]any { + t.Helper() + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + return entry +} + +func TestTimestampFormat(t *testing.T) { + ts := time.Date(2026, 9, 1, 12, 0, 0, 0, time.FixedZone("MSK", 3*3600)) + if got := httpx.Timestamp(ts); got != "2026-09-01T09:00:00.000Z" { + t.Fatalf("Timestamp = %q", got) + } + ts = time.Date(2026, 9, 1, 12, 0, 0, 7_000_000, time.UTC) + if got := httpx.Timestamp(ts); got != "2026-09-01T12:00:00.007Z" { + t.Fatalf("Timestamp = %q", got) + } +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} diff --git a/internal/platform/httpx/identity.go b/internal/platform/httpx/identity.go new file mode 100644 index 00000000..a90fd50c --- /dev/null +++ b/internal/platform/httpx/identity.go @@ -0,0 +1,42 @@ +package httpx + +import ( + "errors" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +const keyUserID = "messenger.user_id" + +// SetUserID records the authenticated caller for the handlers behind the +// authentication middleware and for the access log. +func SetUserID(c *gin.Context, id uuid.UUID) { + c.Set(keyUserID, id) +} + +func UserID(c *gin.Context) (uuid.UUID, bool) { + v, ok := c.Get(keyUserID) + if !ok { + return uuid.Nil, false + } + id, ok := v.(uuid.UUID) + return id, ok +} + +// UserHandler is a handler that needs the authenticated caller. +type UserHandler func(c *gin.Context, actor uuid.UUID) + +// WithUser adapts a UserHandler to gin. Reaching it without the +// authentication middleware in front is a wiring mistake, so it is reported +// as an internal error rather than blamed on the client with a 401. +func WithUser(h UserHandler) gin.HandlerFunc { + return func(c *gin.Context) { + actor, ok := UserID(c) + if !ok { + Error(c, errors.New("route is missing the authentication middleware")) + return + } + h(c, actor) + } +} diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go new file mode 100644 index 00000000..b4844e9d --- /dev/null +++ b/internal/platform/httpx/middleware.go @@ -0,0 +1,131 @@ +package httpx + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "net/http" + "regexp" + "runtime/debug" + "time" + + "github.com/gin-gonic/gin" +) + +// instance names the process that answered, so a client behind the load +// balancer can tell the replicas apart. +func instance(id string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("X-Instance-Id", id) + c.Next() + } +} + +var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// requestID keeps the X-Request-Id set by the load balancer, which overwrites +// any value a client sends; a malformed value is replaced, so a direct caller +// cannot inject into the logs. +func requestID(log *slog.Logger) gin.HandlerFunc { + return func(c *gin.Context) { + id := c.GetHeader("X-Request-Id") + if !requestIDPattern.MatchString(id) { + id = newRequestID() + } + c.Set(keyRequestID, id) + c.Set(keyLogger, log) + c.Writer.Header().Set("X-Request-Id", id) + c.Next() + } +} + +func newRequestID() string { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + panic("httpx: random source unavailable: " + err.Error()) + } + return hex.EncodeToString(raw[:]) +} + +// accessLog writes one line per request at a level that follows the +// outcome (2xx debug, 4xx info, 5xx error), so the ordinary log level decides +// how much traffic is recorded. Headers, query strings and bodies are never +// recorded. time.Since uses the monotonic clock, so wall clock adjustments +// cannot distort the duration. +func accessLog() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + + status := c.Writer.Status() + level := slog.LevelDebug + switch { + case status >= http.StatusInternalServerError: + level = slog.LevelError + case status >= http.StatusBadRequest: + level = slog.LevelInfo + } + log := logger(c) + ctx := c.Request.Context() + if !log.Enabled(ctx, level) { + return + } + attrs := []any{ + "request_id", RequestID(c), + "method", c.Request.Method, + "path", c.Request.URL.Path, + "status", status, + "duration_ms", float64(time.Since(start).Microseconds()) / 1000, + "bytes", c.Writer.Size(), + } + if id, ok := UserID(c); ok { + attrs = append(attrs, "user_id", id.String()) + } + log.Log(ctx, level, "request", attrs...) + } +} + +// recovery turns a panic into a contract 500; the stack goes to the log only. +func recovery() gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + if r := recover(); r != nil { + logger(c).Error("panic recovered", + "request_id", RequestID(c), + "panic", fmt.Sprint(r), + "stack", string(debug.Stack()), + ) + if !c.Writer.Written() { + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + } + c.Abort() + } + }() + c.Next() + } +} + +// Timeout bounds the whole request with one deadline: waiting for a pooled +// connection, SQL statements and session lookups all observe it, so a hung +// dependency ends in a 503 within d instead of holding the client. +func Timeout(d time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), d) + defer cancel() + c.Request = c.Request.WithContext(ctx) + c.Next() + } +} + +// bodyLimit caps request bodies; exceeding it surfaces from the JSON +// decoder as *http.MaxBytesError. +func bodyLimit(maxBytes int64) gin.HandlerFunc { + return func(c *gin.Context) { + if c.Request.Body != nil { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) + } + c.Next() + } +} diff --git a/internal/platform/httpx/respond.go b/internal/platform/httpx/respond.go new file mode 100644 index 00000000..30f49354 --- /dev/null +++ b/internal/platform/httpx/respond.go @@ -0,0 +1,87 @@ +package httpx + +import ( + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +type errorEnvelope struct { + Error errorDetail `json:"error"` +} + +type errorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type statusCode struct { + status int + code string +} + +var kinds = map[apperr.Kind]statusCode{ + apperr.KindInvalid: {http.StatusBadRequest, "invalid_request"}, + apperr.KindUnauthorized: {http.StatusUnauthorized, "unauthorized"}, + apperr.KindNotFound: {http.StatusNotFound, "not_found"}, + apperr.KindConflict: {http.StatusConflict, "conflict"}, + apperr.KindPayloadTooLarge: {http.StatusRequestEntityTooLarge, "payload_too_large"}, + apperr.KindUnsupportedMedia: {http.StatusUnsupportedMediaType, "unsupported_media_type"}, + apperr.KindUnavailable: {http.StatusServiceUnavailable, "unavailable"}, +} + +// JSON is the single place that fixes the response media type; PureJSON +// keeps non-ASCII and HTML characters verbatim. +func JSON(c *gin.Context, status int, body any) { + c.PureJSON(status, body) +} + +// Fail writes the contract error body and stops the handler chain. +func Fail(c *gin.Context, status int, code, message string) { + JSON(c, status, errorEnvelope{Error: errorDetail{ + Code: code, + Message: message, + RequestID: RequestID(c), + }}) + c.Abort() +} + +// Error maps an application error to its status. Anything that is not an +// application error is logged with its text and reported as an opaque 500, +// so internal details never reach the client. An application error shows the +// client only its own message, never the context it was wrapped in on the +// way up; when it is answered with a 5xx (a dependency outage) the full +// chain and the cause go to the log for the operator. +func Error(c *gin.Context, err error) { + appErr, isApp := errors.AsType[*apperr.Error](err) + var mapped statusCode + known := false + if isApp { + mapped, known = kinds[appErr.Kind] + } + if !known { + logger(c).Error("request failed", "request_id", RequestID(c), "error", err.Error()) + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + return + } + if mapped.status >= http.StatusInternalServerError { + attrs := []any{"request_id", RequestID(c), "error", err.Error()} + if appErr.Cause != nil { + attrs = append(attrs, "cause", appErr.Cause.Error()) + } + logger(c).Error("request failed", attrs...) + } + Fail(c, mapped.status, mapped.code, appErr.Message) +} + +// Timestamp renders RFC 3339 UTC with a fixed millisecond width: Python 3.9 +// (the oldest version the public tests support) only parses 3 or 6 fractional +// digits, and RFC3339Nano would trim trailing zeros to a variable width. +func Timestamp(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z") +} diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go new file mode 100644 index 00000000..f50f946a --- /dev/null +++ b/internal/platform/ident/ident.go @@ -0,0 +1,56 @@ +// Package ident generates and parses entity identifiers. +// +// Identifiers are UUIDv7 (RFC 9562): the first 48 bits carry a Unix +// millisecond timestamp and google/uuid keeps them strictly increasing within +// one process even when the system clock steps backwards, so lexicographic +// order of ids equals creation order. +package ident + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "time" + + "github.com/google/uuid" +) + +var ErrInvalid = errors.New("invalid uuid") + +// New panics only if the platform random source is unavailable, which is a +// fatal environment fault rather than a request error. +func New() uuid.UUID { + id, err := uuid.NewV7() + if err != nil { + panic("ident: random source unavailable: " + err.Error()) + } + return id +} + +// NewAlias names one server process for X-Instance-Id; random rather than the +// hostname, so the header reveals nothing about the infrastructure. +func NewAlias() string { + var raw [8]byte + _, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24. + return hex.EncodeToString(raw[:]) +} + +// TimeOf extracts the creation time carried by a UUIDv7 as UTC with +// millisecond precision. +func TimeOf(id uuid.UUID) time.Time { + sec, nsec := id.Time().UnixTime() + return time.Unix(sec, nsec).UTC() +} + +// Parse accepts only the canonical 36-character form; uuid.Parse alone would +// also take braces, urn prefixes and bare hex. +func Parse(raw string) (uuid.UUID, error) { + if len(raw) != 36 { + return uuid.Nil, ErrInvalid + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, ErrInvalid + } + return id, nil +} diff --git a/internal/platform/ident/ident_test.go b/internal/platform/ident/ident_test.go new file mode 100644 index 00000000..d0a57d4d --- /dev/null +++ b/internal/platform/ident/ident_test.go @@ -0,0 +1,84 @@ +package ident_test + +import ( + "bytes" + "regexp" + "testing" + "time" + + "messenger/internal/platform/ident" +) + +func TestNewProducesVersion7(t *testing.T) { + id := ident.New() + if got := id.Version(); got != 7 { + t.Fatalf("version = %d, want 7", got) + } +} + +func TestNewIsStrictlyIncreasingWithinProcess(t *testing.T) { + prev := ident.New() + for i := 0; i < 20000; i++ { + next := ident.New() + if bytes.Compare(next[:], prev[:]) <= 0 { + t.Fatalf("id %s is not greater than previous %s at iteration %d", next, prev, i) + } + prev = next + } +} + +func TestTimeOfMatchesWallClockAtMillisecondPrecision(t *testing.T) { + before := time.Now().Truncate(time.Millisecond) + id := ident.New() + after := time.Now().Add(time.Millisecond) + + got := ident.TimeOf(id) + if got.Location() != time.UTC { + t.Fatalf("TimeOf must return UTC, got %v", got.Location()) + } + if got.Before(before) || got.After(after) { + t.Fatalf("TimeOf = %v, want within [%v, %v]", got, before, after) + } + if got.Nanosecond()%int(time.Millisecond) != 0 { + t.Fatalf("TimeOf must be truncated to milliseconds, got %v", got) + } +} + +func TestParseAcceptsCanonicalForm(t *testing.T) { + id := ident.New() + parsed, err := ident.Parse(id.String()) + if err != nil { + t.Fatalf("Parse(%q) error: %v", id, err) + } + if parsed != id { + t.Fatalf("Parse round trip mismatch: %s != %s", parsed, id) + } +} + +func TestParseRejectsNonCanonicalForms(t *testing.T) { + id := ident.New().String() + for _, raw := range []string{ + "", + "not-a-uuid", + "{" + id + "}", + "urn:uuid:" + id, + id[:32] + id[33:], // hex without one dash + id + "0", + " " + id, + } { + if _, err := ident.Parse(raw); err == nil { + t.Errorf("Parse(%q) must fail", raw) + } + } +} + +func TestNewAliasIs16HexAndRandom(t *testing.T) { + a, b := ident.NewAlias(), ident.NewAlias() + hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`) + if !hex16.MatchString(a) || !hex16.MatchString(b) { + t.Fatalf("aliases %q, %q are not 16 lowercase hex characters", a, b) + } + if a == b { + t.Fatalf("two aliases are equal: %q", a) + } +} diff --git a/internal/platform/pagination/cursor.go b/internal/platform/pagination/cursor.go new file mode 100644 index 00000000..65a09854 --- /dev/null +++ b/internal/platform/pagination/cursor.go @@ -0,0 +1,63 @@ +// Package pagination implements keyset paging. A cursor is opaque to the +// client and bound to the resource it pages (the chat for history, the user +// for the chat list), so a cursor from one resource is rejected on another. +// Because ids are UUIDv7 and creation time derives from the id, the keyset +// (created_at, id) collapses to the id of the last returned item. +package pagination + +import ( + "encoding/base64" + "errors" + "strconv" + + "github.com/google/uuid" +) + +const ( + DefaultLimit = 50 + MaxLimit = 100 +) + +var ( + ErrInvalidCursor = errors.New("invalid cursor") + ErrInvalidLimit = errors.New("limit must be an integer between 1 and 100") +) + +const cursorVersion = 1 + +func EncodeCursor(scope, last uuid.UUID) string { + raw := make([]byte, 0, 1+16+16) + raw = append(raw, cursorVersion) + raw = append(raw, scope[:]...) + raw = append(raw, last[:]...) + return base64.RawURLEncoding.EncodeToString(raw) +} + +func DecodeCursor(token string, scope uuid.UUID) (uuid.UUID, error) { + raw, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(raw) != 1+16+16 || raw[0] != cursorVersion { + return uuid.Nil, ErrInvalidCursor + } + if got, err := uuid.FromBytes(raw[1:17]); err != nil || got != scope { + return uuid.Nil, ErrInvalidCursor + } + last, err := uuid.FromBytes(raw[17:]) + if err != nil { + return uuid.Nil, ErrInvalidCursor + } + return last, nil +} + +// ParseLimit accepts an unsigned decimal between 1 and MaxLimit; an empty +// value means DefaultLimit. ParseUint rejects signs, whitespace and +// fractions and reports overflow, so no length guard is needed. +func ParseLimit(raw string) (int, error) { + if raw == "" { + return DefaultLimit, nil + } + n, err := strconv.ParseUint(raw, 10, 32) + if err != nil || n < 1 || n > MaxLimit { + return 0, ErrInvalidLimit + } + return int(n), nil +} diff --git a/internal/platform/pagination/cursor_test.go b/internal/platform/pagination/cursor_test.go new file mode 100644 index 00000000..5c93a17c --- /dev/null +++ b/internal/platform/pagination/cursor_test.go @@ -0,0 +1,71 @@ +package pagination_test + +import ( + "errors" + "strings" + "testing" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func TestCursorRoundTrip(t *testing.T) { + scope, last := ident.New(), ident.New() + encoded := pagination.EncodeCursor(scope, last) + if encoded == "" || strings.ContainsAny(encoded, "+/=") { + t.Fatalf("cursor must be non-empty URL-safe base64 without padding, got %q", encoded) + } + got, err := pagination.DecodeCursor(encoded, scope) + if err != nil { + t.Fatalf("DecodeCursor error: %v", err) + } + if got != last { + t.Fatalf("decoded %s, want %s", got, last) + } +} + +func TestDecodeCursorRejectsForeignScope(t *testing.T) { + encoded := pagination.EncodeCursor(ident.New(), ident.New()) + _, err := pagination.DecodeCursor(encoded, ident.New()) + if !errors.Is(err, pagination.ErrInvalidCursor) { + t.Fatalf("err = %v, want ErrInvalidCursor", err) + } +} + +func TestDecodeCursorRejectsGarbage(t *testing.T) { + scope := ident.New() + valid := pagination.EncodeCursor(scope, ident.New()) + for _, raw := range []string{"", "***", "abc", valid[:len(valid)-2], valid + "AA", strings.ToUpper(valid)} { + if _, err := pagination.DecodeCursor(raw, scope); !errors.Is(err, pagination.ErrInvalidCursor) { + t.Errorf("DecodeCursor(%q) err = %v, want ErrInvalidCursor", raw, err) + } + } +} + +func TestParseLimit(t *testing.T) { + cases := []struct { + raw string + want int + ok bool + }{ + {"", 50, true}, + {"1", 1, true}, + {"100", 100, true}, + {"0", 0, false}, + {"101", 0, false}, + {"-1", 0, false}, + {"+5", 0, false}, + {"5.0", 0, false}, + {"abc", 0, false}, + {" 5", 0, false}, + } + for _, tc := range cases { + got, err := pagination.ParseLimit(tc.raw) + if tc.ok && (err != nil || got != tc.want) { + t.Errorf("ParseLimit(%q) = %d, %v; want %d", tc.raw, got, err, tc.want) + } + if !tc.ok && !errors.Is(err, pagination.ErrInvalidLimit) { + t.Errorf("ParseLimit(%q) err = %v, want ErrInvalidLimit", tc.raw, err) + } + } +} diff --git a/internal/platform/pagination/page.go b/internal/platform/pagination/page.go new file mode 100644 index 00000000..f905be82 --- /dev/null +++ b/internal/platform/pagination/page.go @@ -0,0 +1,41 @@ +package pagination + +import "github.com/google/uuid" + +// Page is the service-level page: the items and the id to continue from, +// nil when the listing is exhausted. It carries the raw keyset id so that any +// transport can encode the continuation its own way. +type Page[T any] struct { + Items []T + Next *uuid.UUID +} + +// PageResponse is the wire form of a page with the continuation already +// encoded as an opaque cursor. +type PageResponse[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Cut turns a slice over-fetched by one item into a page of at most limit +// items; the extra item only proves that a next page exists. +func Cut[T any](items []T, limit int, id func(T) uuid.UUID) Page[T] { + if len(items) <= limit { + return Page[T]{Items: items} + } + items = items[:limit] + next := id(items[limit-1]) + return Page[T]{Items: items, Next: &next} +} + +func ToResponse[D, R any](p Page[D], scope uuid.UUID, convert func(D) R) PageResponse[R] { + out := PageResponse[R]{Items: make([]R, 0, len(p.Items))} + for _, item := range p.Items { + out.Items = append(out.Items, convert(item)) + } + if p.Next != nil { + cursor := EncodeCursor(scope, *p.Next) + out.NextCursor = &cursor + } + return out +} diff --git a/internal/platform/pagination/page_test.go b/internal/platform/pagination/page_test.go new file mode 100644 index 00000000..6c3b0272 --- /dev/null +++ b/internal/platform/pagination/page_test.go @@ -0,0 +1,60 @@ +package pagination_test + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func self(id uuid.UUID) uuid.UUID { return id } + +func TestCutTrimsOverfetchAndDerivesNext(t *testing.T) { + ids := []uuid.UUID{ident.New(), ident.New(), ident.New()} + + fewer := pagination.Cut(ids[:2], 5, self) + if len(fewer.Items) != 2 || fewer.Next != nil { + t.Fatalf("fewer than limit: %+v", fewer) + } + exact := pagination.Cut(ids[:2], 2, self) + if len(exact.Items) != 2 || exact.Next != nil { + t.Fatalf("exactly limit must not continue: %+v", exact) + } + over := pagination.Cut(ids, 2, self) + if len(over.Items) != 2 || over.Next == nil || *over.Next != ids[1] { + t.Fatalf("over-fetched page must trim and continue from the last kept id: %+v", over) + } + if over.Items[0] != ids[0] || over.Items[1] != ids[1] { + t.Fatalf("order changed: %v", over.Items) + } +} + +func TestToResponseEncodesCursorWithinScope(t *testing.T) { + scope, last := ident.New(), ident.New() + page := pagination.Page[uuid.UUID]{Items: []uuid.UUID{last}, Next: &last} + res := pagination.ToResponse(page, scope, func(id uuid.UUID) string { return id.String() }) + if len(res.Items) != 1 || res.Items[0] != last.String() { + t.Fatalf("items = %v", res.Items) + } + if res.NextCursor == nil { + t.Fatal("next cursor missing") + } + got, err := pagination.DecodeCursor(*res.NextCursor, scope) + if err != nil || got != last { + t.Fatalf("cursor decodes to %s, %v; want %s", got, err, last) + } +} + +func TestToResponseSerializesEmptyPageAsContractRequires(t *testing.T) { + res := pagination.ToResponse(pagination.Page[uuid.UUID]{}, ident.New(), func(id uuid.UUID) string { return id.String() }) + raw, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + if string(raw) != `{"items":[],"next_cursor":null}` { + t.Fatalf("empty page = %s", raw) + } +} diff --git a/internal/platform/password/password.go b/internal/platform/password/password.go new file mode 100644 index 00000000..7c22fdde --- /dev/null +++ b/internal/platform/password/password.go @@ -0,0 +1,74 @@ +// Package password hashes user passwords with Argon2id. +// +// A password has little entropy, so its hash must be slow and memory-hard to +// make offline guessing expensive. Session tokens are the opposite case: they +// are random 256-bit values, and a fast digest is enough for them. +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +// OWASP baseline for Argon2id. The parameters are written into every hash, +// so raising them later does not invalidate stored hashes. +const ( + memoryKiB = 19 * 1024 + iterations = 2 + threads = 1 + saltLen = 16 + keyLen = 32 +) + +var ErrMalformed = errors.New("malformed password hash") + +var b64 = base64.RawStdEncoding + +// Hash returns the PHC string $argon2id$v=19$m=…,t=…,p=…$$. +func Hash(plain string) string { + salt := make([]byte, saltLen) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte(plain), salt, iterations, memoryKiB, threads, keyLen) + return fmt.Sprintf("$argon2id$v=%d$%s$%s$%s", + argon2.Version, params(memoryKiB, iterations, threads), b64.EncodeToString(salt), b64.EncodeToString(key)) +} + +// Verify reports whether plain matches encoded. An error means the stored +// hash itself is unreadable, not that the password is wrong. +func Verify(encoded, plain string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" { + return false, ErrMalformed + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrMalformed + } + var memory, time uint32 + var parallelism uint8 + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, ¶llelism); err != nil || + memory == 0 || time == 0 || parallelism == 0 { + return false, ErrMalformed + } + salt, err := b64.DecodeString(parts[4]) + if err != nil || len(salt) == 0 { + return false, ErrMalformed + } + key, err := b64.DecodeString(parts[5]) + if err != nil || len(key) == 0 { + return false, ErrMalformed + } + got := argon2.IDKey([]byte(plain), salt, time, memory, parallelism, uint32(len(key))) + return subtle.ConstantTimeCompare(got, key) == 1, nil +} + +func params(memory, time uint32, parallelism uint8) string { + return fmt.Sprintf("m=%d,t=%d,p=%d", memory, time, parallelism) +} diff --git a/internal/platform/password/password_test.go b/internal/platform/password/password_test.go new file mode 100644 index 00000000..7c518ad1 --- /dev/null +++ b/internal/platform/password/password_test.go @@ -0,0 +1,78 @@ +package password_test + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "strings" + "testing" + + "golang.org/x/crypto/argon2" + + "messenger/internal/platform/password" +) + +func TestHashVerifiesOnlyTheOriginalPassword(t *testing.T) { + for _, plain := range []string{ + "correct horse battery staple", + "с:двоеточием:и кириллицей", + "emoji 👋👋👋 and : colon", + } { + encoded := password.Hash(plain) + ok, err := password.Verify(encoded, plain) + if err != nil || !ok { + t.Fatalf("Verify(%q) = %v, %v; want true", plain, ok, err) + } + ok, err = password.Verify(encoded, plain+"x") + if err != nil || ok { + t.Fatalf("Verify(wrong) = %v, %v; want false", ok, err) + } + } +} + +func TestHashIsSaltedPHCString(t *testing.T) { + a, b := password.Hash("same password"), password.Hash("same password") + if a == b { + t.Fatal("two hashes of one password are equal: salt is missing") + } + if !strings.HasPrefix(a, "$argon2id$v=19$m=19456,t=2,p=1$") { + t.Fatalf("unexpected encoding %q", a) + } + if strings.Contains(a, "same password") { + t.Fatal("hash contains the plaintext") + } +} + +// A hash made with other parameters must still verify: parameters travel in +// the encoded string, so they can be raised without invalidating old hashes. +func TestVerifyUsesParametersFromTheEncoding(t *testing.T) { + salt := make([]byte, 16) + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte("legacy password"), salt, 1, 8*1024, 2, 32) + b64 := base64.RawStdEncoding + encoded := fmt.Sprintf("$argon2id$v=19$m=8192,t=1,p=2$%s$%s", b64.EncodeToString(salt), b64.EncodeToString(key)) + + ok, err := password.Verify(encoded, "legacy password") + if err != nil || !ok { + t.Fatalf("Verify = %v, %v; want true", ok, err) + } +} + +func TestVerifyRejectsMalformedEncoding(t *testing.T) { + valid := password.Hash("whatever password") + parts := strings.Split(valid, "$") + for name, encoded := range map[string]string{ + "empty": "", + "wrong algo": strings.Replace(valid, "argon2id", "argon2i", 1), + "wrong version": strings.Replace(valid, "v=19", "v=16", 1), + "bad params": strings.Replace(valid, "m=19456,t=2,p=1", "m=x,t=2,p=1", 1), + "zero params": strings.Replace(valid, "m=19456,t=2,p=1", "m=19456,t=0,p=1", 1), + "bad salt": strings.Join([]string{"", parts[1], parts[2], parts[3], "!!!", parts[5]}, "$"), + "bad key": strings.Join([]string{"", parts[1], parts[2], parts[3], parts[4], "!!!"}, "$"), + "missing fields": strings.Join(parts[:5], "$"), + } { + if _, err := password.Verify(encoded, "whatever password"); err == nil { + t.Errorf("%s: Verify accepted %q", name, encoded) + } + } +} diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go new file mode 100644 index 00000000..d9a7d653 --- /dev/null +++ b/internal/platform/postgres/postgres.go @@ -0,0 +1,143 @@ +// Package postgres opens the connection pool and applies the schema. +// +// Server-side timeouts are constants rather than settings: they bound how +// long one request can hold a connection, and the request deadline of the API +// is chosen against them. +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/lock" + + "messenger/internal/platform/apperr" + "messenger/migrations" +) + +const ( + connectTimeout = time.Second + statementTimeout = 5 * time.Second + idleInTxTimeout = 10 * time.Second +) + +const codeUniqueViolation = "23505" + +func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("parse database url: %w", err) + } + cfg.MaxConns = maxConns + cfg.ConnConfig.ConnectTimeout = connectTimeout + // Unlike net.Dialer, pgx resolves the host before ConnectTimeout starts. + // Docker DNS takes seconds to give up on a stopped container: unbounded, + // every request would wait out its deadline and log only "context + // deadline exceeded" instead of the failed lookup. + cfg.ConnConfig.LookupFunc = func(ctx context.Context, host string) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + return net.DefaultResolver.LookupHost(ctx, host) + } + cfg.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(statementTimeout.Milliseconds(), 10) + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = strconv.FormatInt(idleInTxTimeout.Milliseconds(), 10) + // The application keeps every time in UTC; pgx would otherwise scan + // timestamptz into time.Local. + cfg.AfterConnect = func(_ context.Context, conn *pgx.Conn) error { + conn.TypeMap().RegisterType(&pgtype.Type{ + Name: "timestamptz", + OID: pgtype.TimestamptzOID, + Codec: &pgtype.TimestamptzCodec{ScanLocation: time.UTC}, + }) + return nil + } + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("create pool: %w", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + return pool, nil +} + +// Migrate applies pending migrations and returns their versions; a session +// advisory lock makes overlapping runs wait for each other. +func Migrate(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) { + locker, err := lock.NewPostgresSessionLocker() + if err != nil { + return nil, fmt.Errorf("migration lock: %w", err) + } + db := stdlib.OpenDBFromPool(pool) + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.FS, goose.WithSessionLocker(locker)) + if err != nil { + db.Close() + return nil, fmt.Errorf("load migrations: %w", err) + } + defer provider.Close() + results, err := provider.Up(ctx) + if err != nil { + return nil, fmt.Errorf("apply migrations: %w", err) + } + applied := make([]int64, 0, len(results)) + for _, r := range results { + applied = append(applied, r.Source.Version) + } + return applied, nil +} + +// Classify marks a failure to reach the database as apperr.KindUnavailable, +// which the API answers with 503 instead of 500; any other error is returned +// unchanged. Unreachable means: the connection could not be made or broke, +// the request deadline passed, or the server refused work (PostgreSQL error +// classes 08 connection exception, 53 insufficient resources, 57 operator +// intervention such as shutdown or statement timeout). +func Classify(err error) error { + if err == nil || !unreachable(err) { + return err + } + return apperr.Wrap(apperr.KindUnavailable, "database unavailable", err) +} + +func unreachable(err error) bool { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + switch class := pgErr.Code[:min(2, len(pgErr.Code))]; class { + case "08", "53", "57": + return true + } + return false + } + if _, ok := errors.AsType[*pgconn.ConnectError](err); ok { + return true + } + if _, ok := errors.AsType[net.Error](err); ok { + return true + } + return errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, net.ErrClosed) || + pgconn.Timeout(err) || + pgconn.SafeToRetry(err) +} + +// IsUniqueViolation reports whether err is a unique violation of the named +// constraint. +func IsUniqueViolation(err error, constraint string) bool { + pgErr, ok := errors.AsType[*pgconn.PgError](err) + return ok && pgErr.Code == codeUniqueViolation && pgErr.ConstraintName == constraint +} diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go new file mode 100644 index 00000000..6f56fb39 --- /dev/null +++ b/internal/platform/postgres/postgres_test.go @@ -0,0 +1,212 @@ +package postgres_test + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" + "messenger/internal/testenv" + "messenger/migrations" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +func TestPoolScansTimestamptzInUTC(t *testing.T) { + var got time.Time + if err := db.Pool.QueryRow(context.Background(), `SELECT now()`).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Location() != time.UTC { + t.Fatalf("timestamptz scanned in %v, want UTC", got.Location()) + } +} + +func TestMigrateIsRepeatable(t *testing.T) { + ctx := context.Background() + applied, err := postgres.Migrate(ctx, db.Pool) + if err != nil { + t.Fatalf("second migrate: %v", err) + } + if len(applied) != 0 { + t.Fatalf("second migrate applied %v, want nothing", applied) + } + var version int64 + if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil { + t.Fatal(err) + } + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + if version != int64(len(files)) { + t.Fatalf("schema version = %d, want %d (one per migration file)", version, len(files)) + } + for _, table := range []string{"users", "chats", "chat_members", "messages", "message_idempotency_keys"} { + var exists bool + if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("table %s is missing", table) + } + } +} + +func TestPoolAppliesLimitsAndTimeouts(t *testing.T) { + if got := db.Pool.Config().MaxConns; got != 10 { + t.Fatalf("MaxConns = %d, want 10", got) + } + if got := db.Pool.Config().ConnConfig.ConnectTimeout; got != time.Second { + t.Fatalf("ConnectTimeout = %v, want 1s", got) + } + ctx := context.Background() + for setting, want := range map[string]string{ + "statement_timeout": "5s", + "idle_in_transaction_session_timeout": "10s", + } { + var got string + if err := db.Pool.QueryRow(ctx, "SHOW "+setting).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("%s = %q, want %q", setting, got, want) + } + } +} + +func TestResetEmptiesEveryTableButKeepsSchemaVersion(t *testing.T) { + ctx := context.Background() + _, err := db.Pool.Exec(ctx, ` + WITH u AS ( + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000001', 'reset_check', 'Reset', 'x', now()) + RETURNING id + ), c AS ( + INSERT INTO chats (id, title, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000002', 'Reset', now()) + RETURNING id + ), m AS ( + INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM c, u + ) + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT '0190f1d2-0000-7000-8000-000000000003', c.id, u.id, 'hi', now() FROM c, u`) + if err != nil { + t.Fatal(err) + } + + db.Reset(t) + + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + var n int + if err := db.Pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%s has %d rows after Reset", table, n) + } + } + var versions int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM goose_db_version`).Scan(&versions); err != nil { + t.Fatal(err) + } + if versions == 0 { + t.Fatal("Reset wiped the schema version table") + } +} + +func TestIsUniqueViolation(t *testing.T) { + db.Reset(t) + ctx := context.Background() + insert := `INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, 'taken', 'Taken', 'x', now())` + if _, err := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000011"); err != nil { + t.Fatal(err) + } + _, dup := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000012") + if dup == nil { + t.Fatal("duplicate username was accepted") + } + + if !postgres.IsUniqueViolation(dup, "users_username_key") { + t.Fatalf("not recognised as a username violation: %v", dup) + } + if postgres.IsUniqueViolation(dup, "some_other_key") { + t.Fatal("matched a different constraint") + } + if postgres.IsUniqueViolation(fmt.Errorf("wrapped: %w", dup), "users_username_key") == false { + t.Fatal("wrapped error not recognised") + } + if postgres.IsUniqueViolation(errors.New("plain"), "users_username_key") || postgres.IsUniqueViolation(nil, "users_username_key") { + t.Fatal("non-database error recognised as a violation") + } +} + +func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { + unavailable := map[string]error{ + "admin shutdown": &pgconn.PgError{Code: "57P01"}, + "statement timeout": &pgconn.PgError{Code: "57014"}, + "connection failure": &pgconn.PgError{Code: "08006"}, + "too many connections": &pgconn.PgError{Code: "53300"}, + "deadline": fmt.Errorf("query: %w", context.DeadlineExceeded), + "network": &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")}, + "unexpected eof": fmt.Errorf("read: %w", io.ErrUnexpectedEOF), + } + for name, err := range unavailable { + got := postgres.Classify(err) + if apperr.KindOf(got) != apperr.KindUnavailable || !errors.Is(got, err) { + t.Errorf("%s: Classify = %v (kind %v), want KindUnavailable keeping the cause", name, got, apperr.KindOf(got)) + } + } + for name, err := range map[string]error{ + "unique violation": &pgconn.PgError{Code: "23505"}, + "plain": errors.New("boom"), + "nil": nil, + } { + if got := postgres.Classify(err); got != err { + t.Errorf("%s: Classify = %v, want the error unchanged", name, got) + } + } +} + +func TestNewPoolUsesTheRequestedSize(t *testing.T) { + pool, err := postgres.NewPool(context.Background(), db.DSN, 3) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if got := pool.Config().MaxConns; got != 3 { + t.Fatalf("MaxConns = %d, want 3", got) + } +} + +func TestClassifyRealConnectFailure(t *testing.T) { + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1", 1) + if err == nil { + t.Fatal("NewPool to a closed port must fail") + } + if apperr.KindOf(postgres.Classify(err)) != apperr.KindUnavailable { + t.Fatalf("connect failure classified as %v: %v", apperr.KindOf(postgres.Classify(err)), err) + } +} diff --git a/internal/platform/valkey/valkey.go b/internal/platform/valkey/valkey.go new file mode 100644 index 00000000..70c5ba5d --- /dev/null +++ b/internal/platform/valkey/valkey.go @@ -0,0 +1,49 @@ +// Package valkey connects to the Valkey session store. +// +// Timeouts are constants: every authenticated request reads a session, so +// they bound how long a request can wait when the store is slow. Command +// retries are disabled and a connection is dialled once for the same reason: +// a request that cannot reach the store fails fast with 503 instead of +// waiting through a backoff. On the compose stand Docker DNS takes seconds +// to give up on a stopped container, so one dial attempt is already bounded +// only by dialTimeout. +package valkey + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + dialTimeout = time.Second + ioTimeout = time.Second + poolSize = 10 + noRetries = -1 + singleAttempt = 1 +) + +// NewClient connects to url (redis:// scheme) and pings the server. Errors +// never include the URL, which carries the password. +func NewClient(ctx context.Context, url string) (*redis.Client, error) { + opts, err := redis.ParseURL(url) + if err != nil { + return nil, errors.New("session store url is invalid") + } + opts.DialTimeout = dialTimeout + opts.ReadTimeout = ioTimeout + opts.WriteTimeout = ioTimeout + opts.PoolSize = poolSize + opts.MaxRetries = noRetries + opts.DialerRetries = singleAttempt + + client := redis.NewClient(opts) + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, fmt.Errorf("ping session store at %s: %w", opts.Addr, err) + } + return client, nil +} diff --git a/internal/platform/valkey/valkey_test.go b/internal/platform/valkey/valkey_test.go new file mode 100644 index 00000000..efb43ebf --- /dev/null +++ b/internal/platform/valkey/valkey_test.go @@ -0,0 +1,57 @@ +package valkey_test + +import ( + "context" + "strings" + "testing" + "time" + + "messenger/internal/platform/valkey" + "messenger/internal/testenv" +) + +func TestNewClientPingsAndBoundsWaiting(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + client, err := valkey.NewClient(ctx, kv.URL) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer client.Close() + opts := client.Options() + if opts.DialTimeout != time.Second || opts.ReadTimeout != time.Second || opts.WriteTimeout != time.Second { + t.Fatalf("timeouts = %v/%v/%v, want 1s each", opts.DialTimeout, opts.ReadTimeout, opts.WriteTimeout) + } + // go-redis dials up to five times by default; with Docker DNS taking + // seconds to give up on a stopped container that alone would exceed the + // request deadline. + if opts.DialerRetries != 1 { + t.Fatalf("DialerRetries = %d, want a single dial attempt", opts.DialerRetries) + } + // go-redis stores the requested -1 ("no retries") as 0 after init; its + // default would be 3. + if opts.PoolSize != 10 || opts.MaxRetries != 0 { + t.Fatalf("PoolSize = %d, MaxRetries = %d; want 10 and 0 (fail fast)", opts.PoolSize, opts.MaxRetries) + } +} + +func TestNewClientErrorsNeverEchoTheURL(t *testing.T) { + ctx := context.Background() + for name, url := range map[string]string{ + "malformed": "redis://:s3cret@host:port/x", + "unreachable": "redis://:s3cret@127.0.0.1:1/0", + } { + _, err := valkey.NewClient(ctx, url) + if err == nil { + t.Fatalf("%s: want error", name) + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("%s: error leaks the password: %v", name, err) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 00000000..64f7abf8 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,83 @@ +// Package server runs the HTTP listener with a graceful stop behind a load +// balancer. +package server + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "time" +) + +// Readiness is switched off when shutdown starts. +type Readiness interface { + SetReady(ready bool) +} + +// Run serves until ctx is cancelled, keeps serving for drainDelay with +// readiness off, then shuts down within shutdownTimeout. The balancer learns +// of the stop only from failing readiness checks, so drainDelay must exceed +// its detection time. +func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 64 * 1024, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + + served := make(chan error, 1) + go func() { + err := srv.Serve(ln) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + served <- err + }() + log.Info("listening", "addr", ln.Addr().String()) + + select { + case err := <-served: + return err + case <-ctx.Done(): + } + + log.Info("shutdown started", "drain_delay", drainDelay.String(), "timeout", shutdownTimeout.String()) + ready.SetReady(false) + time.Sleep(drainDelay) + drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := srv.Shutdown(drain); err != nil { + log.Error("shutdown forced", "error", err.Error()) + _ = srv.Close() + return err + } + <-served + log.Info("shutdown completed") + return nil +} + +// RunDebug serves the internal metrics until ctx is cancelled; no request +// depends on them, so it stops without draining. +func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + go func() { + <-ctx.Done() + _ = srv.Close() + }() + log.Info("debug listening", "addr", ln.Addr().String()) + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 00000000..8a2caa39 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,169 @@ +package server_test + +import ( + "context" + "io" + "log/slog" + "net" + "net/http" + "testing" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/health" + "messenger/internal/platform/httpx" + "messenger/internal/server" +) + +// newHandler builds the smallest handler with probes; the lifecycle does not +// depend on the business modules or their stores. +func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { + e := httpx.NewEngine(log, 1024, "test") + probes := health.New() + probes.Register(e) + return e, probes +} + +func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() + + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop within 10s after cancel") + } + if _, err := http.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after shutdown") + } +} + +func TestReadinessDropsBeforeListenerCloses(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + // Hold one request open so shutdown has something to drain. + slow := make(chan struct{}) + go func() { + resp, err := http.Get(base + "/health/live") + if err == nil { + resp.Body.Close() + } + close(slow) + }() + + cancel() + <-done + <-slow + if !probes.IsReady() { + return + } + t.Fatal("readiness must be switched off during shutdown") +} + +// During the drain delay the balancer is still learning that the replica is +// going away: readiness already fails, yet new connections are accepted and +// served. Only after the delay does the listener close. +func TestDrainKeepsServingWhileNotReady(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + const drain = 700 * time.Millisecond + go func() { done <- server.Run(ctx, log, ln, h, probes, drain, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + waitFor(t, base+"/health/ready", http.StatusServiceUnavailable) + // A fresh connection, as the balancer opens for a new request. + fresh := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + resp, err := fresh.Get(base + "/health/live") + if err != nil { + t.Fatalf("new connection during the drain delay: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("liveness during drain = %d, want 200", resp.StatusCode) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop after the drain delay") + } + if _, err := fresh.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after the drain delay") + } +} + +func waitFor(t *testing.T, url string, status int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + if resp.StatusCode == status { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not return %d in time", url, status) +} + +// The debug listener has no drain: it serves the benchmark's metrics scraper +// only and stops with the process. +func TestRunDebugServesUntilCancelled(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.RunDebug(ctx, log, ln, h) }() + waitFor(t, "http://"+ln.Addr().String()+"/debug/vars", http.StatusTeapot) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunDebug returned %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunDebug did not stop after cancel") + } +} diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go new file mode 100644 index 00000000..57cf53bd --- /dev/null +++ b/internal/testenv/testenv.go @@ -0,0 +1,181 @@ +// Package testenv starts disposable PostgreSQL and Valkey containers for tests +// that exercise the real stores. Only _test files import it, so it never +// reaches the server binary. The images match compose.yaml. +package testenv + +import ( + "context" + "fmt" + "os" + "runtime" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcvalkey "github.com/testcontainers/testcontainers-go/modules/valkey" + + "messenger/internal/platform/postgres" + "messenger/internal/platform/valkey" +) + +const ( + PostgresImage = "postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873" + ValkeyImage = "valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11" +) + +// useDockerDesktopPipe names the Docker Desktop engine pipe explicitly on +// Windows. Without DOCKER_HOST, testcontainers probes the pipe with a file +// stat, which fails while the pipe is busy; with several test packages +// starting containers at once that probe fails spuriously and the run panics. +// Dialling the named host instead waits for a free pipe instance. +var useDockerDesktopPipe = sync.OnceFunc(func() { + if runtime.GOOS == "windows" && os.Getenv("DOCKER_HOST") == "" { + _ = os.Setenv("DOCKER_HOST", "npipe:////./pipe/docker_engine") + } +}) + +// DB is a migrated PostgreSQL instance. +type DB struct { + Pool *pgxpool.Pool + DSN string + container *tcpostgres.PostgresContainer +} + +func StartPostgres(ctx context.Context) (*DB, error) { + useDockerDesktopPipe() + c, err := tcpostgres.Run(ctx, PostgresImage, + tcpostgres.WithDatabase("messenger"), + tcpostgres.WithUsername("messenger"), + tcpostgres.WithPassword("messenger"), + tcpostgres.BasicWaitStrategies(), + ) + if err != nil { + return nil, fmt.Errorf("start postgres: %w", err) + } + db := &DB{container: c} + db.DSN, err = c.ConnectionString(ctx, "sslmode=disable") + if err != nil { + db.Close() + return nil, fmt.Errorf("postgres address: %w", err) + } + db.Pool, err = postgres.NewPool(ctx, db.DSN, 10) + if err != nil { + db.Close() + return nil, err + } + if _, err := postgres.Migrate(ctx, db.Pool); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// Reset empties every application table, so each test starts from an empty +// schema. The table list comes from the catalog: a table added by a later +// migration cannot be forgotten here. +func (d *DB) Reset(t testing.TB) { + t.Helper() + ctx := context.Background() + var tables string + err := d.Pool.QueryRow(ctx, ` + SELECT coalesce(string_agg(format('%I.%I', schemaname, tablename), ', '), '') + FROM pg_tables + WHERE schemaname = 'public' AND tablename <> 'goose_db_version'`).Scan(&tables) + if err != nil { + t.Fatalf("list tables: %v", err) + } + if tables == "" { + return + } + if _, err := d.Pool.Exec(ctx, "TRUNCATE "+tables+" CASCADE"); err != nil { + t.Fatalf("truncate: %v", err) + } +} + +// Stop stops the container, as when the database goes down. +func (d *DB) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return d.container.Stop(ctx, &timeout) +} + +// Pause freezes the server: connections stay open but nothing answers, as +// with a hung database. +func (d *DB) Pause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerPause(ctx, d.container.GetContainerID()) + }) +} + +func (d *DB) Unpause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerUnpause(ctx, d.container.GetContainerID()) + }) +} + +func withDocker(ctx context.Context, fn func(cli *testcontainers.DockerClient) error) error { + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + return err + } + defer cli.Close() + return fn(cli) +} + +func (d *DB) Close() { + if d.Pool != nil { + d.Pool.Close() + } + _ = testcontainers.TerminateContainer(d.container) +} + +// KV is a Valkey instance without persistence or password. +type KV struct { + Client *redis.Client + URL string + container *tcvalkey.ValkeyContainer +} + +func StartValkey(ctx context.Context) (*KV, error) { + useDockerDesktopPipe() + c, err := tcvalkey.Run(ctx, ValkeyImage) + if err != nil { + return nil, fmt.Errorf("start valkey: %w", err) + } + kv := &KV{container: c} + kv.URL, err = c.ConnectionString(ctx) + if err != nil { + kv.Close() + return nil, fmt.Errorf("valkey address: %w", err) + } + // The production client, so outage tests see its fail-fast settings. + kv.Client, err = valkey.NewClient(ctx, kv.URL) + if err != nil { + kv.Close() + return nil, err + } + return kv, nil +} + +func (k *KV) Reset(t testing.TB) { + t.Helper() + if err := k.Client.FlushDB(context.Background()).Err(); err != nil { + t.Fatalf("flushdb: %v", err) + } +} + +// Stop stops the container, as when the session store goes down. +func (k *KV) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return k.container.Stop(ctx, &timeout) +} + +func (k *KV) Close() { + if k.Client != nil { + _ = k.Client.Close() + } + _ = testcontainers.TerminateContainer(k.container) +} diff --git a/internal/testenv/testenv_test.go b/internal/testenv/testenv_test.go new file mode 100644 index 00000000..6488bded --- /dev/null +++ b/internal/testenv/testenv_test.go @@ -0,0 +1,29 @@ +package testenv_test + +import ( + "context" + "testing" + + "messenger/internal/testenv" +) + +func TestValkeyStartsAndResetFlushesKeys(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + if err := kv.Client.Set(ctx, "probe", "1", 0).Err(); err != nil { + t.Fatal(err) + } + kv.Reset(t) + n, err := kv.Client.DBSize(ctx).Result() + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%d keys left after Reset", n) + } +} diff --git a/internal/user/handler.go b/internal/user/handler.go new file mode 100644 index 00000000..a5f091af --- /dev/null +++ b/internal/user/handler.go @@ -0,0 +1,70 @@ +package user + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/httpx" +) + +type Handler struct { + users *Service +} + +func NewHandler(users *Service) *Handler { + return &Handler{users: users} +} + +// Register mounts registration on the public router and the current-user +// endpoint on the router guarded by the authentication middleware. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/users", h.register) + protected.GET("/users/me", httpx.WithUser(h.me)) +} + +type createRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type Response struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +func ToResponse(u User) Response { + return Response{ + ID: u.ID.String(), + Username: u.Username, + DisplayName: u.DisplayName, + CreatedAt: httpx.Timestamp(u.CreatedAt), + } +} + +func (h *Handler) register(c *gin.Context) { + var req createRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + u, err := h.users.Register(c.Request.Context(), req.Username, req.DisplayName, req.Password) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, ToResponse(u)) +} + +func (h *Handler) me(c *gin.Context, actor uuid.UUID) { + u, err := h.users.GetByID(c.Request.Context(), actor) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, ToResponse(u)) +} diff --git a/internal/user/main_test.go b/internal/user/main_test.go new file mode 100644 index 00000000..ba3a1ca2 --- /dev/null +++ b/internal/user/main_test.go @@ -0,0 +1,24 @@ +package user_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} diff --git a/internal/user/repository.go b/internal/user/repository.go new file mode 100644 index 00000000..eb2ac2f6 --- /dev/null +++ b/internal/user/repository.go @@ -0,0 +1,18 @@ +package user + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores a new user or returns ErrUsernameTaken; the uniqueness + // check and the insert must be atomic. + Create(ctx context.Context, user User) error + GetByID(ctx context.Context, id uuid.UUID) (User, error) + GetByUsername(ctx context.Context, username string) (User, error) + // ExistingIDs returns the subset of ids that belong to registered users, + // in no particular order, in one round trip. + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} diff --git a/internal/user/service.go b/internal/user/service.go new file mode 100644 index 00000000..708ddb9b --- /dev/null +++ b/internal/user/service.go @@ -0,0 +1,88 @@ +package user + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/password" +) + +type Service struct { + repo Repository +} + +func NewService(repo Repository) *Service { + return &Service{repo: repo} +} + +// Register validates every field before hashing, so invalid input never pays +// for the KDF. +func (s *Service) Register(ctx context.Context, username, displayName, plain string) (User, error) { + u, err := New(ident.New(), username, displayName) + if err != nil { + return User{}, err + } + if err := ValidatePassword(plain); err != nil { + return User{}, err + } + u.PasswordHash = password.Hash(plain) + if err := s.repo.Create(ctx, u); err != nil { + return User{}, err + } + return u, nil +} + +// dummyHash is verified for unknown usernames, so a failed login takes the +// same time whether or not the user exists. +var dummyHash = sync.OnceValue(func() string { + return password.Hash("timing-equaliser-password") +}) + +// Authenticate returns the user whose password matches, or +// ErrInvalidCredentials for an unknown username and a wrong password alike. +// The username comes straight from the Basic header, so one that cannot +// exist (including NUL or invalid UTF-8, which the database would reject) +// is answered without a query, but after the same KDF work. +func (s *Service) Authenticate(ctx context.Context, username, plain string) (User, error) { + if !usernamePattern.MatchString(username) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + u, err := s.repo.GetByUsername(ctx, username) + if errors.Is(err, ErrNotFound) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + if err != nil { + return User{}, fmt.Errorf("find user: %w", err) + } + ok, err := password.Verify(u.PasswordHash, plain) + if err != nil { + return User{}, fmt.Errorf("verify password of user %s: %w", u.ID, err) + } + if !ok { + return User{}, ErrInvalidCredentials + } + return u, nil +} + +func (s *Service) GetByID(ctx context.Context, id uuid.UUID) (User, error) { + return s.repo.GetByID(ctx, id) +} + +// ExistingIDs serves the chats module when it validates member lists. +func (s *Service) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + if len(ids) == 0 { + return []uuid.UUID{}, nil + } + found, err := s.repo.ExistingIDs(ctx, ids) + if err != nil { + return nil, fmt.Errorf("user lookup: %w", err) + } + return found, nil +} diff --git a/internal/user/service_test.go b/internal/user/service_test.go new file mode 100644 index 00000000..3406ddd8 --- /dev/null +++ b/internal/user/service_test.go @@ -0,0 +1,189 @@ +package user_test + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" + "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +// newService wires the service to the real store over an empty database. +func newService(t *testing.T) *user.Service { + t.Helper() + db.Reset(t) + return user.NewService(store.NewUsers(db.Pool)) +} + +func TestRegisterStoresUserWithPasswordHash(t *testing.T) { + svc := newService(t) + ctx := context.Background() + u, err := svc.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatalf("Register error: %v", err) + } + if u.ID.Version() != 7 { + t.Fatalf("id must be UUIDv7, got version %d", u.ID.Version()) + } + if !strings.HasPrefix(u.PasswordHash, "$argon2id$") || strings.Contains(u.PasswordHash, pw) { + t.Fatalf("password must be stored as an Argon2id hash, got %q", u.PasswordHash) + } + got, err := svc.GetByID(ctx, u.ID) + if err != nil { + t.Fatalf("GetByID error: %v", err) + } + if got != u { + t.Fatalf("GetByID = %+v, want %+v", got, u) + } +} + +func TestRegisterRejectsInvalidInputAndStoresNothing(t *testing.T) { + svc := newService(t) + ctx := context.Background() + cases := map[string][3]string{ + "username": {"Bad Name", "x", pw}, + "display_name": {"alice", " ", pw}, + "password": {"alice", "Алиса", "short"}, + } + for field, in := range cases { + _, err := svc.Register(ctx, in[0], in[1], in[2]) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), field) { + t.Errorf("%s: err = %v, want KindInvalid naming the field", field, err) + } + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&n); err != nil || n != 0 { + t.Fatalf("users = %d, %v; invalid input must not be stored", n, err) + } +} + +func TestRegisterReportsDuplicateAsConflict(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "A", pw); err != nil { + t.Fatal(err) + } + _, err := svc.Register(ctx, "alice", "B", pw) + if !errors.Is(err, user.ErrUsernameTaken) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("err = %v, want ErrUsernameTaken with KindConflict", err) + } +} + +// Concurrent registrations of one username race on the unique constraint: +// exactly one wins and every other gets the conflict, never a 500. +func TestConcurrentRegistrationOfOneUsernameHasOneWinner(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const attempts = 10 + errs := make(chan error, attempts) + var start, done sync.WaitGroup + start.Add(1) + for range attempts { + done.Add(1) + go func() { + defer done.Done() + start.Wait() + _, err := svc.Register(ctx, "racer", "Racer", pw) + errs <- err + }() + } + start.Done() + done.Wait() + close(errs) + + won, lost := 0, 0 + for err := range errs { + switch { + case err == nil: + won++ + case errors.Is(err, user.ErrUsernameTaken): + lost++ + default: + t.Errorf("unexpected error: %v", err) + } + } + if won != 1 || lost != attempts-1 { + t.Fatalf("won = %d, lost = %d; want 1 and %d", won, lost, attempts-1) + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users WHERE username = 'racer'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("rows = %d, %v; want exactly one", n, err) + } +} + +func TestGetByIDUnknownIsNotFound(t *testing.T) { + svc := newService(t) + _, err := svc.GetByID(context.Background(), ident.New()) + if !errors.Is(err, user.ErrNotFound) || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("err = %v, want ErrNotFound with KindNotFound", err) + } +} + +func TestExistingIDsReturnsOnlyRegistered(t *testing.T) { + svc := newService(t) + ctx := context.Background() + alice, _ := svc.Register(ctx, "alice", "A", pw) + bob, _ := svc.Register(ctx, "bob", "B", pw) + got, err := svc.ExistingIDs(ctx, []uuid.UUID{ident.New(), bob.ID, alice.ID}) + if err != nil { + t.Fatal(err) + } + set := map[uuid.UUID]bool{} + for _, id := range got { + set[id] = true + } + if len(got) != 2 || !set[alice.ID] || !set[bob.ID] { + t.Fatalf("ExistingIDs = %v, want alice and bob", got) + } + if empty, err := svc.ExistingIDs(ctx, nil); err != nil || len(empty) != 0 { + t.Fatalf("ExistingIDs(nil) = %v, %v", empty, err) + } +} + +func TestAuthenticateChecksPassword(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const colonPassword = "пароль:с:двоеточием 👋" + alice, err := svc.Register(ctx, "alice", "Алиса", colonPassword) + if err != nil { + t.Fatal(err) + } + got, err := svc.Authenticate(ctx, "alice", colonPassword) + if err != nil || got != alice { + t.Fatalf("Authenticate = %+v, %v; want alice", got, err) + } +} + +// A wrong password and an unknown username must be indistinguishable. +func TestAuthenticateFailuresLookTheSame(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "Алиса", pw); err != nil { + t.Fatal(err) + } + _, wrong := svc.Authenticate(ctx, "alice", pw+"!") + _, unknown := svc.Authenticate(ctx, "nobody", pw) + // Basic credentials are client bytes: a username the database cannot + // even compare must still be a plain credential failure. + _, nul := svc.Authenticate(ctx, "a\x00b", pw) + _, notUTF8 := svc.Authenticate(ctx, "\xff\xfe", pw) + for name, err := range map[string]error{ + "wrong password": wrong, "unknown user": unknown, "nul username": nul, "invalid utf-8 username": notUTF8, + } { + if !errors.Is(err, user.ErrInvalidCredentials) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if wrong.Error() != unknown.Error() { + t.Fatalf("messages differ: %q vs %q", wrong, unknown) + } +} diff --git a/internal/user/store/postgres.go b/internal/user/store/postgres.go new file mode 100644 index 00000000..d8974847 --- /dev/null +++ b/internal/user/store/postgres.go @@ -0,0 +1,74 @@ +// Package store holds the repository implementations of the users module. +// Errors pass through postgres.Classify, so an unreachable database surfaces +// as unavailability rather than as an internal error. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/platform/postgres" + "messenger/internal/user" +) + +type Users struct { + pool *pgxpool.Pool +} + +var _ user.Repository = (*Users)(nil) + +func NewUsers(pool *pgxpool.Pool) *Users { + return &Users{pool: pool} +} + +// Create relies on the unique constraint rather than a prior lookup, so two +// concurrent registrations of one username cannot both succeed. +func (r *Users) Create(ctx context.Context, u user.User) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES ($1, $2, $3, $4, $5)`, + u.ID, u.Username, u.DisplayName, u.PasswordHash, u.CreatedAt) + if postgres.IsUniqueViolation(err, "users_username_key") { + return user.ErrUsernameTaken + } + if err != nil { + return fmt.Errorf("insert user: %w", postgres.Classify(err)) + } + return nil +} + +func (r *Users) GetByID(ctx context.Context, id uuid.UUID) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1`, id) +} + +func (r *Users) GetByUsername(ctx context.Context, username string) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1`, username) +} + +func (r *Users) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + rows, err := r.pool.Query(ctx, `SELECT id FROM users WHERE id = ANY($1)`, ids) + if err != nil { + return nil, fmt.Errorf("select user ids: %w", postgres.Classify(err)) + } + found, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) + if err != nil { + return nil, fmt.Errorf("read user ids: %w", postgres.Classify(err)) + } + return found, nil +} + +func (r *Users) one(ctx context.Context, query string, arg any) (user.User, error) { + var u user.User + err := r.pool.QueryRow(ctx, query, arg).Scan(&u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return user.User{}, user.ErrNotFound + } + if err != nil { + return user.User{}, fmt.Errorf("select user: %w", postgres.Classify(err)) + } + return u, nil +} diff --git a/internal/user/user.go b/internal/user/user.go new file mode 100644 index 00000000..d05e1c43 --- /dev/null +++ b/internal/user/user.go @@ -0,0 +1,75 @@ +// Package user is the users module. The entity mirrors the table row and +// carries no behaviour; New only enforces input invariants and CreatedAt +// derives from the UUIDv7 id. The password itself is never kept: the service +// validates it and stores only its hash. +package user + +import ( + "fmt" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxDisplayNameCodePoints = 100 + MinPasswordCodePoints = 12 + MaxPasswordCodePoints = 128 +) + +var ( + ErrNotFound = apperr.NotFound("user not found") + ErrUsernameTaken = apperr.Conflict("username already taken") + // ErrInvalidCredentials covers an unknown username and a wrong password + // alike, so a login attempt cannot probe which usernames exist. + ErrInvalidCredentials = apperr.Unauthorized("invalid username or password") +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +func New(id uuid.UUID, username, displayName string) (User, error) { + if !usernamePattern.MatchString(username) { + return User{}, apperr.Invalid("username must match [a-z0-9_]{3,32}") + } + if strings.TrimSpace(displayName) == "" { + return User{}, apperr.Invalid("display_name must not be blank") + } + if utf8.RuneCountInString(displayName) > MaxDisplayNameCodePoints { + return User{}, apperr.Invalid(fmt.Sprintf("display_name must be at most %d characters", MaxDisplayNameCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(displayName, 0) { + return User{}, apperr.Invalid("display_name must not contain the NUL character") + } + return User{ + ID: id, + Username: username, + DisplayName: displayName, + CreatedAt: ident.TimeOf(id), + }, nil +} + +// ValidatePassword checks the contract length in Unicode code points. +func ValidatePassword(plain string) error { + if strings.TrimSpace(plain) == "" { + return apperr.Invalid("password must not be blank") + } + if n := utf8.RuneCountInString(plain); n < MinPasswordCodePoints || n > MaxPasswordCodePoints { + return apperr.Invalid(fmt.Sprintf("password must be %d to %d characters", MinPasswordCodePoints, MaxPasswordCodePoints)) + } + return nil +} diff --git a/internal/user/user_test.go b/internal/user/user_test.go new file mode 100644 index 00000000..ede9ec52 --- /dev/null +++ b/internal/user/user_test.go @@ -0,0 +1,78 @@ +package user_test + +import ( + "strings" + "testing" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" +) + +func TestNewDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + u, err := user.New(id, "alice", "Алиса") + if err != nil { + t.Fatalf("New error: %v", err) + } + if u.ID != id || u.Username != "alice" || u.DisplayName != "Алиса" { + t.Fatalf("unexpected user %+v", u) + } + if !u.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", u.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewValidatesUsername(t *testing.T) { + valid := []string{"abc", "a_1", strings.Repeat("a", 32), "user_2026"} + for _, name := range valid { + if _, err := user.New(ident.New(), name, "ok"); err != nil { + t.Errorf("username %q must be accepted: %v", name, err) + } + } + invalid := []string{"", "ab", strings.Repeat("a", 33), "Alice", "al ice", "алиса", "a-b", "a.b", " abc", "abc\n"} + for _, name := range invalid { + _, err := user.New(ident.New(), name, "ok") + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "username") { + t.Errorf("username %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} + +// The contract counts the password length in Unicode code points, not bytes. +func TestValidatePasswordCountsCodePoints(t *testing.T) { + valid := []string{ + strings.Repeat("я", 12), + strings.Repeat("👋", 128), + "pass:word:with:colons", + " spaced but long enough ", + } + for _, p := range valid { + if err := user.ValidatePassword(p); err != nil { + t.Errorf("password of %d code points must be accepted: %v", len([]rune(p)), err) + } + } + invalid := []string{"", strings.Repeat(" ", 12), strings.Repeat("\t", 20), strings.Repeat("я", 11), strings.Repeat("👋", 129)} + for _, p := range invalid { + err := user.ValidatePassword(p) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "password") { + t.Errorf("password %q must be rejected as invalid naming the field, got %v", p, err) + } + } +} + +func TestNewValidatesDisplayNameByCodePoints(t *testing.T) { + valid := []string{"A", "Алиса", strings.Repeat("я", 100), "👋 hi"} + for _, name := range valid { + if _, err := user.New(ident.New(), "alice", name); err != nil { + t.Errorf("display_name %q must be accepted: %v", name, err) + } + } + invalid := []string{"", " ", "\t\n", strings.Repeat("я", 101), "nul\x00inside"} + for _, name := range invalid { + _, err := user.New(ident.New(), "alice", name) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "display_name") { + t.Errorf("display_name %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} diff --git a/migrations/00001_init.sql b/migrations/00001_init.sql new file mode 100644 index 00000000..400b762e --- /dev/null +++ b/migrations/00001_init.sql @@ -0,0 +1,42 @@ +-- +goose Up +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL CONSTRAINT users_username_key UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +-- Lists the chats of one user in (created_at, id) order: chat ids are UUIDv7. +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +-- The key is (chat_id, id) and serves the history query directly: +-- WHERE chat_id = $1 AND id > $2 ORDER BY id LIMIT $3. Nothing looks a +-- message up by id alone, and a separate index on id would only cost writes +-- and tempt the planner into scanning every chat's messages in id order. +CREATE TABLE messages ( + chat_id uuid NOT NULL REFERENCES chats (id), + id uuid NOT NULL, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, id) +); + +-- +goose Down +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; diff --git a/migrations/00002_message_idempotency_keys.sql b/migrations/00002_message_idempotency_keys.sql new file mode 100644 index 00000000..6fe78f6f --- /dev/null +++ b/migrations/00002_message_idempotency_keys.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method + +-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys. +CREATE TABLE message_idempotency_keys ( + user_id uuid NOT NULL, + chat_id uuid NOT NULL, + key text NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key), + FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id) +); + +CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at); + +-- +goose Down +DROP TABLE message_idempotency_keys; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 00000000..29af9ab0 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +// Package migrations embeds the versioned SQL schema, so the server binary +// always carries the schema it was built against. +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS diff --git a/requirements-bench.txt b/requirements-bench.txt new file mode 100644 index 00000000..b1ec8530 --- /dev/null +++ b/requirements-bench.txt @@ -0,0 +1,2 @@ +# Plotting of the track 1 results (scripts/bench/plot.py); install into a venv. +matplotlib==3.10.3 diff --git a/scripts/api-instances.sh b/scripts/api-instances.sh new file mode 100755 index 00000000..be6ce307 --- /dev/null +++ b/scripts/api-instances.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Lists the API containers: . The +# alias is random per process, so each running replica is asked over +# /health/live, in parallel: the failover hook must return within 10 seconds. +set -eu +cd "$(dirname "$0")/.." + +ids=$(docker compose ps -aq api) +[ -n "$ids" ] || exit 0 +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +docker inspect --format '{{.Id}} {{.Name}} {{.State.Status}} {{.Image}}' $ids >"$tmp/containers" +while read -r id name state image; do + if [ "$state" = running ]; then + # wget prints the response headers on stderr; the body is dropped. + (docker exec "$id" wget -S -q -O - http://127.0.0.1:8080/health/live 2>&1 >/dev/null | + sed -n 's/^ *[Xx]-[Ii]nstance-[Ii]d: *//p' | tr -d '\r' | head -n 1 >"$tmp/$id") & + fi +done <"$tmp/containers" +wait + +while read -r id name state image; do + alias=- + if [ -s "$tmp/$id" ]; then + alias=$(cat "$tmp/$id") + fi + image=${image#sha256:} + printf '%s %s %s %s %s\n' "$(printf '%s' "$id" | cut -c1-12)" "${name#/}" "$state" "$alias" "$(printf '%s' "$image" | cut -c1-12)" +done <"$tmp/containers" diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh new file mode 100755 index 00000000..afff54d9 --- /dev/null +++ b/scripts/bench/failover.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and +# started again at RESTORE_AT, under a constant write load that retries +# uncertain sends with the same key; then every confirmed message must be in +# the history once. +# +# sh scripts/bench/failover.sh stop|kill +# +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT +# (evidence/failover). Times come from the Docker VM clock; the Windows clock +# differs by a fraction of a second and is kept for reference only. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +kind=${1:?usage: failover.sh stop|kill } +n=${2:?usage: failover.sh stop|kill } +case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac +rate=${RATE:-50} +duration=${DURATION:-60s} +fault_at=${FAULT_AT:-15} +restore_at=${RESTORE_AT:-35} +dir=${OUT:-evidence/failover}/$kind-$n +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +target=$(docker compose ps -q api | sed -n "${n}p") +[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; } +alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }') +meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ + "fault_at_s=$fault_at" "restore_at_s=$restore_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$fault_at" +fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker "$kind" "$target" >/dev/null +exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target") +sleep $((restore_at - fault_at)) +restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker compose start api >/dev/null 2>&1 +started=$(docker inspect --format '{{.State.StartedAt}}' "$target") +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json" +docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh new file mode 100644 index 00000000..7ea1be60 --- /dev/null +++ b/scripts/bench/lib.sh @@ -0,0 +1,77 @@ +# Shared helpers of the experiment scripts; sourced, not run. + +# The generator and the knobs the experiments vary live in compose.bench.yaml. +export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml +# The balancer logs errors and state changes only, not every request. +export LB_QUIET=1 + +# abspath prints a directory in the form the Docker CLI mounts (a Windows path +# under Git Bash). +abspath() { + (cd "$1" && (pwd -W 2>/dev/null || pwd)) +} + +# meta writes what ran, when, on which commit and host: file, then key=value. +meta() { + file=$1 + shift + { + printf '{\n' + printf ' "date_utc": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf ' "commit": "%s",\n' "$(git rev-parse --short HEAD)" + printf ' "branch": "%s",\n' "$(git branch --show-current)" + printf ' "docker": "%s",\n' "$(docker version --format '{{.Server.Version}}')" + printf ' "host_cpus": "%s",\n' "$(docker info --format '{{.NCPU}}')" + printf ' "host_memory_bytes": "%s",\n' "$(docker info --format '{{.MemTotal}}')" + for pair in "$@"; do + printf ' "%s": "%s",\n' "${pair%%=*}" "${pair#*=}" + done + printf ' "go": "%s"\n' "$(go env GOVERSION 2>/dev/null || echo unknown)" + printf '}\n' + } >"$file" +} + +# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: +# the same work every time, so a slow spell of the host shows in the results. +host_probe() { + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab4 \ + -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' +} + +# state_log keeps from `compose logs` (stdin) what explains a run: the +# balancer's server state changes and the replicas' start and stop. An +# overloaded run logs every failed request, gigabytes over a series; those +# lines are only counted. +state_log() { + awk '/is DOWN|is UP|UP\/READY|maintenance|"msg":"(starting|shutdown started|shutdown completed)"/ { print; next } + { dropped++ } END { printf "# %d other lines dropped\n", dropped }' +} + +# sample_docker_stats appends the CPU and memory of every container until killed. +sample_docker_stats() { + while :; do + docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ + $(docker ps -q --filter label=com.docker.compose.project=lab4) 2>/dev/null || true + done >>"$1" +} + +# sample_postgres appends, every second, the database counters and what the +# active sessions wait for ("cpu": nothing). +sample_postgres() { + while :; do + docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " + SELECT json_build_object( + 't', extract(epoch FROM now()), + 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, + 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'sessions', sessions, 'sessions_abandoned', sessions_abandoned, + 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), + 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( + SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n + FROM pg_stat_activity + WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid() + GROUP BY 1) AS x)) + FROM pg_stat_database WHERE datname = 'messenger'" 2>/dev/null || true + sleep 1 + done >>"$1" +} diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py new file mode 100755 index 00000000..5258974c --- /dev/null +++ b/scripts/bench/plot.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Tables and charts of track 1 from the raw results of run-series.sh. + + python scripts/bench/plot.py + +The raw root holds //rep/ directories. Written to the +output directory: + summary.csv one row per step of every run; + capacity.csv the capacity of every run and the median per configuration; + control.csv the noise checks of every run, the host speed probe and the + loadgen CPU per request at the first step, each flagged when + more than 25 % off its median; + cost.csv CPU milliseconds per request of each component at the + COST_RATES steps (valid, error-free, unflagged runs only), and + the rate one database CPU would serve at that cost; + -throughput.png, -p99.png, capacity.png, + cpu-at-capacity.png, pool-waits.png, cost-per-request.png. +Capacity is the highest step that passed the criterion loadgen applies (valid, +under 1 % errors, p99 of successful requests within 500 ms). Charts show the +median of the repeats; a hollow marker means the generator fell behind. +""" +import csv +import gzip +import json +import pathlib +import statistics +import sys +from datetime import datetime + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +SLO_P99_MS = 500.0 +CONTROL_TOLERANCE = 0.25 +# Below saturation in every configuration, so the cost is not inflated by +# queueing. +COST_RATES = (800, 1000) +COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") + + +def read_jsonl(path): + packed = path.with_name(path.name + ".gz") + if packed.exists(): + text = gzip.decompress(packed.read_bytes()).decode("utf-8") + elif path.exists(): + text = path.read_text(encoding="utf-8") + else: + return [] + rows = [] + for line in text.splitlines(): + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass + return rows + + +def component(name): + for c in COMPONENTS: + if f"-{c}-" in name: + return c + return None + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def load_runs(root): + runs = [] + for steps_path in sorted(root.glob("*/*/rep*/steps.json")): + d = steps_path.parent + report = json.loads(steps_path.read_text(encoding="utf-8")) + runs.append({ + "series": d.parent.parent.name, + "config": d.parent.name, + "rep": d.name, + "start": parse_time(report["started_at"]), + "steps": report["steps"], + "docker": read_jsonl(d / "docker-stats.jsonl"), + "metrics": read_jsonl(d / "metrics.jsonl"), + "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, + }) + return runs + + +def mean_cpu(run, t0, t1): + """Mean CPU per component in [t0, t1); replicas of the API are summed.""" + per_time = {} + for row in run["docker"]: + c = component(row.get("name", "")) + if c is None or not t0 <= row["t"] < t1: + continue + try: + cpu = float(row["cpu"].rstrip("%")) + except (ValueError, KeyError): + continue + per_time.setdefault(row["t"], {}).setdefault(c, 0.0) + per_time[row["t"]][c] += cpu + out = {} + for c in COMPONENTS: + values = [v[c] for v in per_time.values() if c in v] + out[c] = round(statistics.mean(values), 1) if values else "" + return out + + +def pool_rate(run, t0, t1, counter): + """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" + first, last = {}, {} + for row in run["metrics"]: + t = run["start"] + row["t"] + if not t0 <= t < t1: + continue + for replica in row.get("replicas") or []: + pool = replica.get("pgxpool") or {} + key = replica.get("instance") + if key and counter in pool: + first.setdefault(key, pool[counter]) + last[key] = pool[counter] + if not last: + return "" + return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) + + +def main(root, out): + runs = load_runs(root) + if not runs: + sys.exit(f"no runs under {root}") + out.mkdir(parents=True, exist_ok=True) + + rows, capacities = [], [] + for run in runs: + best = None + for step in run["steps"]: + t0 = run["start"] + step["started_at_s"] + t1 = t0 + step["seconds"] + cpu = mean_cpu(run, t0, t1) + classes = step["classes"] + row = { + "series": run["series"], "config": run["config"], "rep": run["rep"], + "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), + "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], + "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], + "client_timeouts": classes.get("client_timeout", 0), + "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), + "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), + **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, + "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), + "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), + } + rows.append(row) + if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): + best = row + capacities.append({ + "series": run["series"], "config": run["config"], "rep": run["rep"], + "capacity_rps": best["offered_rps"] if best else 0, "at": best, + "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), + }) + + with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + configs = [] + for c in capacities: + if (c["series"], c["config"]) not in configs: + configs.append((c["series"], c["config"])) + with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) + medians = {} + for key in configs: + mine = [c for c in capacities if (c["series"], c["config"]) == key] + values = [c["capacity_rps"] for c in mine] + medians[key] = statistics.median(values) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", + sum(c["invalid_below"] for c in mine)]) + + control = [] + for run in runs: + first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] + if first and first[0]["cpu_loadgen_pct"] != "": + probe = run["meta"].get("host_probe_s", "") + control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, + round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) + if control: + probes = [c[3] for c in control if c[3] is not None] + probe_median = statistics.median(probes) if probes else None + cpu_median = statistics.median(c[4] for c in control) + with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", + "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) + flagged = set() + for series, config, rep, probe, cpu in control: + probe_dev = probe / probe_median - 1 if probe is not None else 0.0 + cpu_dev = cpu / cpu_median - 1 + noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE + if noisy: + flagged.add((series, config, rep)) + w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, + f"{cpu_dev:+.2f}", noisy]) + + cost = {} + for r in rows: + if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES: + continue + if not r["valid"] or r["error_share"] > 0: + continue + per = cost.setdefault((r["series"], r["config"]), {"runs": set()}) + per["runs"].add(r["rep"]) + for c in ("api", "postgres", "lb"): + if r[f"cpu_{c}_pct"] != "": + per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"]) + with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request", + "lb_cpu_ms_per_request", "rps_per_database_cpu"]) + for (series, config), per in cost.items(): + med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)} + w.writerow([series, config, " ".join(sorted(per["runs"])), + *(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")), + round(1000 / med["postgres"]) if med.get("postgres") else ""]) + if cost: + keys = list(cost) + fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5)) + width = 0.27 + for i, c in enumerate(("api", "postgres", "lb")): + ax.bar([j + i * width for j in range(len(keys))], + [statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c) + ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30) + ax.set_ylabel("CPU ms per request") + ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cost-per-request.png", dpi=120) + plt.close(fig) + + def median_by_offered(series, config, field): + points, valid = {}, {} + for r in rows: + if r["series"] == series and r["config"] == config and r[field] != "": + points.setdefault(r["offered_rps"], []).append(float(r[field])) + valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] + xs = sorted(points) + return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] + + for series in sorted({s for s, _ in configs}): + names = [c for s, c in configs if s == series] + for field, title, filename, log in ( + ("ok_rps", "Successful requests per second", "throughput", False), + ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), + ): + fig, ax = plt.subplots(figsize=(7, 4.5)) + top = 0 + for name in names: + xs, ys, valid = median_by_offered(series, name, field) + line, = ax.plot(xs, ys, label=name) + ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) + ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], + facecolors="none", edgecolors=line.get_color()) + top = max([top, *xs]) + if field == "ok_rps" and top: + ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") + if field == "ok_p99_ms": + ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") + if log: + ax.set_yscale("log") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel(title) + ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / f"{series}-{filename}.png", dpi=120) + plt.close(fig) + + labels = [f"{s}/{c}" for s, c in configs] + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) + meds = [medians[k] for k in configs] + lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] + highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] + ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") + ax.set_ylabel("Capacity, requests per second") + ax.set_title("Capacity per configuration (median, range of repeats)") + ax.tick_params(axis="x", rotation=30) + ax.grid(axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(out / "capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) + width = 0.8 / len(COMPONENTS) + for i, comp in enumerate(COMPONENTS): + values = [] + for key in configs: + at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] + cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] + values.append(statistics.median(cpu) if cpu else 0) + ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) + ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) + ax.set_ylabel("Mean CPU at capacity, % of one core") + ax.set_title("Which component is busy at capacity") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cpu-at-capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(7, 4.5)) + for series, name in configs: + xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") + if xs: + ax.plot(xs, ys, marker="o", label=f"{series}/{name}") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel("Waits for a pooled connection per second") + ax.set_title("Connection pool waits, all replicas") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "pool-waits.png", dpi=120) + plt.close(fig) + print(f"{len(runs)} runs, {len(rows)} steps -> {out}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])) diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh new file mode 100755 index 00000000..44cfb40b --- /dev/null +++ b/scripts/bench/rollout-under-load.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a +# constant write load that retries uncertain sends with the same key; then +# every confirmed message must be in the history once. +# +# sh scripts/bench/rollout-under-load.sh +# +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout). +# Times come from the Docker VM clock, as in failover.sh. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +rate=${RATE:-50} +duration=${DURATION:-60s} +update_at=${UPDATE_AT:-15} +dir=${OUT:-evidence/rollout} +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +# Built beforehand, as by a delivery pipeline: the update replaces replicas. +docker compose build api >/dev/null 2>&1 +meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) +old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ') + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$update_at" +started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +status=0 +sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? +finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +new="" +for id in $(docker compose ps -q api); do + new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, " +done +printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \ + "$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json" +docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh new file mode 100755 index 00000000..f291fa77 --- /dev/null +++ b/scripts/bench/run-series.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# Track 1: every configuration of the given series files, each on a fresh +# stand, under one grid of offered rates. Repeats are interleaved (repeat 1 +# of every configuration, then repeat 2), so a slow spell of the host does not +# fall on one configuration. +# +# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...] +# +# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s), +# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT +# (evidence/track1/raw). Results: ///rep/. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh ..." >&2; exit 2; } +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}" +repeats=${REPEATS:-3} +step=${STEP:-30s} +warmup=${WARMUP:-30s} +out=${OUT:-evidence/track1/raw} +stop= +[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail + +# Fixed for every configuration; the API, its pool, maxconn and the database +# CPU come from the series line. +export LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 +export MSYS_NO_PATHCONV=1 + +# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus. +run_config() { + series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10} + dir=$out/$series/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \ + DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus + + docker compose down -v --remove-orphans >/dev/null 2>&1 + if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then + echo " the stand did not start, see $dir/up.log" >&2 + exit 1 + fi + meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \ + "db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \ + "lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \ + "stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)" + + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + # stdin closed: `compose run` would otherwise read it. + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 /dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + docker compose logs --no-color --timestamps lb api 2>&1 | state_log >"$dir/compose.log" + gzip -9 "$dir"/*.jsonl + grep '^step' "$dir/loadgen.log" || true +} + +for conf in "$@"; do + [ -r "$conf" ] || { echo "cannot read $conf" >&2; exit 2; } + bad=$(grep -v '^#' "$conf" | awk 'NF && NF != 8') + [ -z "$bad" ] || { echo "$conf: a line needs 8 fields: $bad" >&2; exit 2; } +done +docker compose --profile bench build loadgen >/dev/null +default_ifs=$IFS +newline=' +' +for rep in $(seq 1 "$repeats"); do + for conf in "$@"; do + lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d') + IFS=$newline + for line in $lines; do + IFS=$default_ifs + # shellcheck disable=SC2086 # the line splits into the fields + run_config "$(basename "$conf" .conf)" "$rep" $line + done + IFS=$default_ifs + done +done +docker compose down -v --remove-orphans >/dev/null 2>&1 +echo "results in $out" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf new file mode 100644 index 00000000..a461f2b3 --- /dev/null +++ b/scripts/bench/series/A.conf @@ -0,0 +1,6 @@ +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas; +# the pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A1 1 2.0 1024M 920MiB 10 0 1.0 +A2 2 1.0 512M 460MiB 10 0 1.0 +A4 4 0.5 256M 230MiB 10 0 1.0 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf new file mode 100644 index 00000000..6ebe2902 --- /dev/null +++ b/scripts/bench/series/Aprime.conf @@ -0,0 +1,5 @@ +# Series A': the budget of series A with the total pool fixed at 20 connections; +# A'2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +Ap1 1 2.0 1024M 920MiB 20 0 1.0 +Ap4 4 0.5 256M 230MiB 5 0 1.0 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf new file mode 100644 index 00000000..9f4fc11c --- /dev/null +++ b/scripts/bench/series/B.conf @@ -0,0 +1,5 @@ +# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +B1 1 1.0 512M 460MiB 10 0 1.0 +B3 3 1.0 512M 460MiB 10 0 1.0 +B4 4 1.0 512M 460MiB 10 0 1.0 diff --git a/scripts/bench/series/C.conf b/scripts/bench/series/C.conf new file mode 100644 index 00000000..20d012f3 --- /dev/null +++ b/scripts/bench/series/C.conf @@ -0,0 +1,10 @@ +# Improvement C: maxconn per replica on the balancer, before (0, no limit) and +# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0); +# 10 and 40 show how much the value matters. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +R1m0 1 1.0 512M 460MiB 10 0 1.0 +R1m20 1 1.0 512M 460MiB 10 20 1.0 +R4m0 4 1.0 512M 460MiB 10 0 1.0 +R4m20 4 1.0 512M 460MiB 10 20 1.0 +R1m10 1 1.0 512M 460MiB 10 10 1.0 +R1m40 1 1.0 512M 460MiB 10 40 1.0 diff --git a/scripts/bench/series/control.conf b/scripts/bench/series/control.conf new file mode 100644 index 00000000..76a08eec --- /dev/null +++ b/scripts/bench/series/control.conf @@ -0,0 +1,6 @@ +# Control: A2 next to A2 with twice the database CPU, interleaved, run with +# STOP_ON_FAIL=0 so that one noisy step does not end a ladder. If the database +# is the bottleneck, the capacity rises. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A2 2 1.0 512M 460MiB 10 0 1.0 +A2pg2 2 1.0 512M 460MiB 10 0 2.0 diff --git a/scripts/explain-history.sql b/scripts/explain-history.sql new file mode 100644 index 00000000..f455eedd --- /dev/null +++ b/scripts/explain-history.sql @@ -0,0 +1,75 @@ +-- EXPLAIN evidence for the message history query (make explain). +-- +-- Everything runs in one transaction that is rolled back: it seeds 12 000 +-- messages in one chat and 90 000 in 300 other chats, so the plan is measured +-- on a realistic volume without leaving data on the stand. The PREPARE text +-- is the statement of Messages.ListByChat in internal/chat/store/postgres.go; +-- keep them identical. +\set ON_ERROR_STOP on +\pset pager off +BEGIN; + +SELECT uuidv7() AS author \gset +SELECT uuidv7() AS big_chat \gset + +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES (:'author', 'explain_' || substr(md5(random()::text), 1, 12), 'EXPLAIN', 'not-a-password-hash', + uuid_extract_timestamp(:'author')); + +CREATE TEMP TABLE noise_chats ON COMMIT DROP AS +SELECT uuidv7() AS id FROM generate_series(1, 300); + +INSERT INTO chats (id, title, created_at) +SELECT :'big_chat'::uuid, 'explain: big chat', uuid_extract_timestamp(:'big_chat') +UNION ALL +SELECT id, 'explain: other chat', uuid_extract_timestamp(id) FROM noise_chats; + +INSERT INTO chat_members (chat_id, user_id) +SELECT id, :'author' FROM chats WHERE title LIKE 'explain:%'; + +-- Rows are inserted ordered by n, so the messages of all chats interleave in +-- the heap as they would under real traffic; ids stay increasing per chat. +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT id, chat_id, :'author', 'message ' || n, uuid_extract_timestamp(id) +FROM ( + SELECT uuidv7() AS id, chat_id, n + FROM ( + SELECT :'big_chat'::uuid AS chat_id, n FROM generate_series(1, 12000) AS n + UNION ALL + SELECT c.id, n FROM noise_chats AS c, generate_series(1, 300) AS n + ) AS planned + ORDER BY n +) AS generated; + +ANALYZE users, chats, chat_members, messages; + +SELECT (SELECT count(*) FROM messages WHERE chat_id = :'big_chat') AS big_chat_messages, + (SELECT count(*) FROM messages) AS all_messages; + +PREPARE history(uuid, uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 + AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY id +LIMIT $3; + +\echo +\echo == First page: no cursor, limit 50 (+1 row that tells whether a next page exists) +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', NULL, 51); + +SELECT id AS deep_cursor FROM messages WHERE chat_id = :'big_chat' ORDER BY id OFFSET 10000 LIMIT 1 \gset +\echo +\echo == Deep page: cursor after the 10 001st message of the chat +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', :'deep_cursor', 51); + +\echo +\echo == For contrast, OFFSET pagination of the same page reads every skipped row +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = :'big_chat' +ORDER BY id +OFFSET 10001 LIMIT 51; + +ROLLBACK; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..ae7df48b --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Creates the local credentials of the compose stand in ./secrets, which git +# ignores. Existing secrets are kept: PostgreSQL applies its password only +# when the volume is first initialised, so a new password would lock the API +# out of an existing database. To start over: make destroy, then delete +# ./secrets and run this again. +set -eu +cd "$(dirname "$0")/.." + +files="postgres_password database_url valkey_auth.conf session_store_url" +present=0 +for f in $files; do + if [ -e "secrets/$f" ]; then + present=$((present + 1)) + fi +done +if [ "$present" -eq 4 ]; then + echo "secrets already exist in ./secrets, kept" + exit 0 +fi +if [ "$present" -ne 0 ]; then + echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2 + exit 1 +fi + +# 24 random bytes as hex: URL-safe without escaping. +random() { + od -An -tx1 -N24 /dev/urandom | tr -d ' \n' +} +pg=$(random) +kv=$(random) + +# The directory is private; the files stay readable for the unprivileged +# users inside the containers, which see them through bind mounts. +umask 022 +mkdir -p secrets +chmod 700 secrets +printf '%s\n' "$pg" > secrets/postgres_password +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url +printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf +printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url +echo "created ./secrets" diff --git a/scripts/instances.sh b/scripts/instances.sh new file mode 100755 index 00000000..02fbbefd --- /dev/null +++ b/scripts/instances.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Which container answers under which X-Instance-Id. +set -eu +cd "$(dirname "$0")/.." + +printf '%-12s %-14s %-8s %-16s %s\n' CONTAINER NAME STATE X-INSTANCE-ID IMAGE +sh scripts/api-instances.sh | while read -r id name state alias image; do + printf '%-12s %-14s %-8s %-16s %s\n' "$id" "$name" "$state" "$alias" "$image" +done diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..5f86f138 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/restart.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0restart.sh" +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..7eda3b6a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Restart hook of `make test-persistence`: stops the whole stand gracefully and +# starts the same containers again; volumes are kept. +set -eu +cd "$(dirname "$0")/.." +docker compose stop +docker compose up -d --wait diff --git a/scripts/rollout.sh b/scripts/rollout.sh new file mode 100755 index 00000000..4710e64c --- /dev/null +++ b/scripts/rollout.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout, +# MIT): new replicas start next to the old ones, and the old ones go once all +# new ones are healthy; if one is not, the new ones are removed instead. +# +# With a stopped replica docker rollout would replace only that one and leave +# two versions running, so the script refuses. The migrate container is +# recreated from the new image without starting: the `up` of docker rollout +# then applies the new migrations once, before any new replica starts. +set -eu +cd "$(dirname "$0")/.." + +idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api) +if [ -n "$idle" ]; then + echo "some API replicas are not running; start them first: docker compose start api" >&2 + exit 2 +fi + +version=v0.14 +checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b +tool=bin/docker-rollout + +if ! printf '%s %s\n' "$checksum" "$tool" | sha256sum -c - >/dev/null 2>&1; then + mkdir -p bin + curl -fsSL -o "$tool.download" "https://github.com/wowu/docker-rollout/releases/download/$version/docker-rollout" + if ! printf '%s %s\n' "$checksum" "$tool.download" | sha256sum -c - >/dev/null 2>&1; then + rm -f "$tool.download" + echo "docker-rollout $version: checksum mismatch, not used" >&2 + exit 1 + fi + mv "$tool.download" "$tool" +fi + +docker compose build api +docker compose up --no-start --no-deps --force-recreate migrate +# Run through sh: Windows cannot use the script as a docker CLI plugin. +if ! sh "$tool" rollout --wait-after-healthy 4 api; then + # A failed migration leaves the new containers created but not started. + created=$(docker compose ps -aq --status created api) + if [ -n "$created" ]; then + docker rm $created >/dev/null + fi + exit 1 +fi diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..76322518 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/stop-one.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0stop-one.sh" +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..6971e1b4 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Failover hook of `make test-failover`: gracefully stops the one running API +# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else. +# Restore it with: docker compose start api +set -eu +cd "$(dirname "$0")/.." + +target=${TARGET_INSTANCE_ID:-} +case "$target" in +'' | *[!0-9a-f]*) + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 + ;; +esac +if [ ${#target} -ne 16 ]; then + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 +fi + +matches=$(sh scripts/api-instances.sh | awk -v t="$target" '$3 == "running" && $4 == t { print $1 }') +count=$(printf '%s\n' "$matches" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo "expected one running API replica with X-Instance-Id $target, found $count" >&2 + exit 2 +fi +docker stop "$matches" >/dev/null +echo "stopped API container $matches (X-Instance-Id $target)" diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go new file mode 100644 index 00000000..9b583e0d --- /dev/null +++ b/tools/concurrency/client.go @@ -0,0 +1,135 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" +) + +// client is a minimal typed HTTP client for the messenger API. +type client struct { + base string + http *http.Client +} + +func newClient(baseURL string) *client { + return &client{base: baseURL, http: &http.Client{Timeout: 10 * time.Second}} +} + +func (c *client) post(path, token string, body, out any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+path, bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, token, http.StatusCreated, out) +} + +// postMessage sends one message under a fresh Idempotency-Key. +func (c *client) postMessage(chatID, token, text string, out any) error { + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", uuid.NewString()) + return c.do(req, token, http.StatusCreated, out) +} + +// sendWithKey sends one attempt on a connection of its own, so that the +// attempts of one key reach different replicas. +func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { + var msg chat.MessageResponse + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return 0, msg, "", err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return 0, msg, "", err + } + req.Close = true + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Idempotency-Key", key) + resp, err := c.http.Do(req) + if err != nil { + return 0, msg, "", err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return 0, msg, "", err + } + if resp.StatusCode == http.StatusCreated { + if err := json.Unmarshal(payload, &msg); err != nil { + return 0, msg, "", err + } + } + return resp.StatusCode, msg, resp.Header.Get("X-Instance-Id"), nil +} + +// login opens a session with Basic credentials and returns its bearer token. +func (c *client) login(username, password string) (string, error) { + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + req.SetBasicAuth(username, password) + var session struct { + Token string `json:"token"` + } + if err := c.do(req, "", http.StatusCreated, &session); err != nil { + return "", err + } + return session.Token, nil +} + +// get requests the largest page the contract allows, continuing from cursor +// when it is set. +func (c *client) get(path, token, cursor string, out any) error { + query := url.Values{"limit": {"100"}} + if cursor != "" { + query.Set("cursor", cursor) + } + req, err := http.NewRequest(http.MethodGet, c.base+path+"?"+query.Encode(), nil) + if err != nil { + return err + } + return c.do(req, token, http.StatusOK, out) +} + +func (c *client) do(req *http.Request, token string, want int, out any) error { + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return err + } + if resp.StatusCode != want { + return fmt.Errorf("%s %s: status %d, want %d: %s", req.Method, req.URL.Path, resp.StatusCode, want, payload) + } + return json.Unmarshal(payload, out) +} diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go new file mode 100644 index 00000000..a1ac25e6 --- /dev/null +++ b/tools/concurrency/main.go @@ -0,0 +1,322 @@ +// Command concurrency is the black-box race check from the grading rubric. +// +// Phase 1: N clients send M messages each into one chat at the same time, +// then the history is paged and must contain exactly N*M unique messages in +// id order. +// +// Phase 2: in each of K rounds one key is sent P times in parallel through the +// balancer; every attempt must get 201 with one message, at least two replicas +// must answer, the history must hold K messages, and another text must get 409. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/user" +) + +// Report is the machine readable outcome saved as evidence. +type Report struct { + BaseURL string `json:"base_url"` + StartedAt time.Time `json:"started_at"` + Clients int `json:"clients"` + PerClient int `json:"per_client"` + Expected int `json:"expected_messages"` + Created int `json:"created_201"` + Failed int `json:"failed_posts"` + Listed int `json:"listed_messages"` + Unique int `json:"unique_ids"` + Missing int `json:"created_but_not_listed"` + Ordered bool `json:"history_ordered_by_id"` + Pages int `json:"pages_read"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + // Idempotency is phase 2. + Idempotency IdempotencyReport `json:"idempotency"` + Passed bool `json:"passed"` +} + +type IdempotencyReport struct { + Rounds int `json:"rounds"` + Parallel int `json:"parallel"` + // SameMessageRounds counts rounds whose attempts all answered 201 with + // one and the same message. + SameMessageRounds int `json:"same_message_rounds"` + // InstancesSeen is the number of distinct X-Instance-Id values among all + // attempts. + InstancesSeen int `json:"instances_seen"` + RequireReplicas bool `json:"require_two_replicas"` + Conflict409 bool `json:"different_body_409"` + History int `json:"history_messages"` + Passed bool `json:"passed"` +} + +func main() { + baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + clients := flag.Int("clients", 10, "number of concurrent clients") + perClient := flag.Int("per-client", 10, "messages sent by each client") + rounds := flag.Int("rounds", 20, "idempotency rounds, one key each") + parallel := flag.Int("parallel", 8, "parallel attempts with the same key per round") + requireReplicas := flag.Bool("require-replicas", true, "fail unless the attempts reached at least two replicas") + output := flag.String("output", "", "optional path for the JSON report") + flag.Parse() + + report, err := run(*baseURL, *clients, *perClient, *rounds, *parallel, *requireReplicas) + if err != nil { + fail(err) + } + text, _ := json.MarshalIndent(report, "", " ") + fmt.Println(string(text)) + if *output != "" { + if err := writeReport(*output, text); err != nil { + fail(err) + } + } + if !report.Passed { + fail(fmt.Errorf("concurrency invariants violated")) + } + fmt.Printf("PASS: %d clients x %d messages produced exactly %d unique ordered messages\n", + report.Clients, report.PerClient, report.Unique) + fmt.Printf("PASS: %d rounds x %d parallel attempts with one key produced %d messages across %d instances\n", + report.Idempotency.Rounds, report.Idempotency.Parallel, report.Idempotency.History, report.Idempotency.InstancesSeen) +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "FAIL:", err) + os.Exit(1) +} + +func writeReport(path string, text []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, append(text, '\n'), 0o644) +} + +func run(baseURL string, clients, perClient, rounds, parallel int, requireReplicas bool) (Report, error) { + if clients < 2 || perClient < 1 || rounds < 1 || parallel < 1 { + return Report{}, fmt.Errorf("clients must be at least 2; per-client, rounds and parallel positive") + } + report := Report{ + BaseURL: baseURL, + StartedAt: time.Now().UTC(), + Clients: clients, + PerClient: perClient, + Expected: clients * perClient, + } + api := newClient(baseURL) + runID := uuid.NewString()[:8] + + members, tokens, err := registerClients(api, runID, clients) + if err != nil { + return report, err + } + room, err := createChat(api, runID, members, tokens[0]) + if err != nil { + return report, err + } + + started := time.Now() + created, failed := sendConcurrently(baseURL, room.ID, tokens, perClient) + report.ElapsedSeconds = time.Since(started).Seconds() + report.Created = len(created) + report.Failed = failed + + listed, pages, err := readHistory(api, room.ID, tokens[0]) + if err != nil { + return report, err + } + report.Pages = pages + report.Listed = len(listed) + report.Unique, report.Missing = compare(created, listed) + report.Ordered = sort.StringsAreSorted(listed) + report.Idempotency, err = checkIdempotency(api, baseURL, runID, members[:2], tokens[0], rounds, parallel, requireReplicas) + if err != nil { + return report, err + } + report.Passed = report.Failed == 0 && + report.Created == report.Expected && + report.Listed == report.Expected && + report.Unique == report.Expected && + report.Missing == 0 && + report.Ordered && + report.Idempotency.Passed + return report, nil +} + +// checkIdempotency runs phase 2 in a chat of its own. +func checkIdempotency(api *client, baseURL, runID string, members []string, token string, rounds, parallel int, requireReplicas bool) (IdempotencyReport, error) { + r := IdempotencyReport{Rounds: rounds, Parallel: parallel, RequireReplicas: requireReplicas} + room, err := createChat(api, runID+"-idem", members, token) + if err != nil { + return r, err + } + instances := map[string]bool{} + var key, text string + for round := range rounds { + key, text = uuid.NewString(), fmt.Sprintf("idempotency round %d", round) + type attempt struct { + status int + id string + instance string + } + attempts := make([]attempt, parallel) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range parallel { + wg.Go(func() { + own := newClient(baseURL) + <-start + status, msg, instance, err := own.sendWithKey(room.ID, token, key, text) + if err == nil { + attempts[i] = attempt{status, msg.ID, instance} + } + }) + } + close(start) + wg.Wait() + same := true + for _, a := range attempts { + if a.instance != "" { + instances[a.instance] = true + } + if a.status != http.StatusCreated || a.id == "" || a.id != attempts[0].id { + same = false + } + } + if same { + r.SameMessageRounds++ + } + } + r.InstancesSeen = len(instances) + status, _, _, err := newClient(baseURL).sendWithKey(room.ID, token, key, text+" changed") + if err != nil { + return r, err + } + r.Conflict409 = status == http.StatusConflict + listed, _, err := readHistory(api, room.ID, token) + if err != nil { + return r, err + } + r.History = len(listed) + r.Passed = r.SameMessageRounds == rounds && + r.History == rounds && + r.Conflict409 && + (!requireReplicas || r.InstancesSeen >= 2) + return r, nil +} + +// clientPassword is shared by the synthetic clients of one run. +const clientPassword = "concurrency-check-password" + +// registerClients registers and logs in n clients, returning their ids and +// bearer tokens in the same order. +func registerClients(api *client, runID string, n int) (ids, tokens []string, err error) { + for i := range n { + var created user.Response + username := fmt.Sprintf("load_%s_%d", runID, i) + body := map[string]string{ + "username": username, + "display_name": fmt.Sprintf("Load client %d", i), + "password": clientPassword, + } + if err := api.post("/api/v1/users", "", body, &created); err != nil { + return nil, nil, fmt.Errorf("register client %d: %w", i, err) + } + token, err := api.login(username, clientPassword) + if err != nil { + return nil, nil, fmt.Errorf("log in client %d: %w", i, err) + } + ids = append(ids, created.ID) + tokens = append(tokens, token) + } + return ids, tokens, nil +} + +func createChat(api *client, runID string, members []string, creatorToken string) (chat.ChatResponse, error) { + var created chat.ChatResponse + body := map[string]any{"title": "Concurrency check " + runID, "member_ids": members} + if err := api.post("/api/v1/chats", creatorToken, body, &created); err != nil { + return chat.ChatResponse{}, fmt.Errorf("create chat: %w", err) + } + return created, nil +} + +// sendConcurrently gives every member its own HTTP client and releases them +// together, so the server sees genuinely parallel writes to one chat. +func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (map[string]struct{}, int) { + var mu sync.Mutex + created := make(map[string]struct{}, len(tokens)*perClient) + failed := 0 + + var wg sync.WaitGroup + start := make(chan struct{}) + for i, token := range tokens { + wg.Add(1) + go func(n int, token string) { + defer wg.Done() + own := newClient(baseURL) + <-start + for m := range perClient { + var msg chat.MessageResponse + err := own.postMessage(chatID, token, fmt.Sprintf("client %d message %d", n, m), &msg) + mu.Lock() + if err != nil { + failed++ + } else { + created[msg.ID] = struct{}{} + } + mu.Unlock() + } + }(i, token) + } + close(start) + wg.Wait() + return created, failed +} + +func readHistory(api *client, chatID, token string) ([]string, int, error) { + var ids []string + pages := 0 + cursor := "" + for { + var page pagination.PageResponse[chat.MessageResponse] + if err := api.get("/api/v1/chats/"+chatID+"/messages", token, cursor, &page); err != nil { + return nil, pages, fmt.Errorf("read history: %w", err) + } + pages++ + for _, m := range page.Items { + ids = append(ids, m.ID) + } + if page.NextCursor == nil { + return ids, pages, nil + } + cursor = *page.NextCursor + } +} + +// compare returns how many distinct ids the history holds and how many +// acknowledged ids never showed up in it. +func compare(created map[string]struct{}, listed []string) (unique, missing int) { + seen := make(map[string]struct{}, len(listed)) + for _, id := range listed { + seen[id] = struct{}{} + } + for id := range created { + if _, ok := seen[id]; !ok { + missing++ + } + } + return len(seen), missing +} diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go new file mode 100644 index 00000000..f0e1b045 --- /dev/null +++ b/tools/concurrency/main_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "os" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func TestRunAgainstLiveServer(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 10, 10, 5, 4, false) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Clients != 10 || report.PerClient != 10 || report.Expected != 100 { + t.Fatalf("report = %+v", report) + } + if report.Created != 100 || report.Listed != 100 || report.Unique != 100 || report.Missing != 0 { + t.Fatalf("report = %+v", report) + } + if !report.Ordered || !report.Passed { + t.Fatalf("report = %+v", report) + } + idem := report.Idempotency + if idem.Rounds != 5 || idem.Parallel != 4 || idem.SameMessageRounds != 5 || idem.History != 5 || !idem.Conflict409 || !idem.Passed { + t.Fatalf("idempotency = %+v", idem) + } + if idem.InstancesSeen != 1 { + t.Fatalf("one in-process server must be seen as one instance: %+v", idem) + } +} + +// Through the balancer the same key must reach at least two replicas; one +// replica alone does not prove the idempotency is shared. +func TestIdempotencyRequiresTwoReplicasByDefault(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 2, 1, 2, 2, true) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Idempotency.Passed || report.Passed { + t.Fatalf("a single instance passed the shared idempotency check: %+v", report.Idempotency) + } +} + +func TestRunFailsOnUnreachableServer(t *testing.T) { + if _, err := run("http://127.0.0.1:1", 2, 2, 1, 1, false); err == nil { + t.Fatal("expected an error for an unreachable server") + } +} diff --git a/tools/loadgen/client.go b/tools/loadgen/client.go new file mode 100644 index 00000000..fce2c380 --- /dev/null +++ b/tools/loadgen/client.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// result is one HTTP exchange; instance is the X-Instance-Id of the answer, +// empty when the balancer answered itself or nothing answered. +type result struct { + status int + instance string + body []byte + latency time.Duration + err error +} + +type client struct { + base string + http *http.Client + timeout time.Duration +} + +// newClient keeps many connections open to the balancer, as many concurrent +// users would; HAProxy balances every request, not every connection. +func newClient(base string, timeout time.Duration) *client { + transport := &http.Transport{ + MaxIdleConns: 10000, + MaxIdleConnsPerHost: 10000, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + } + return &client{base: base, http: &http.Client{Transport: transport}, timeout: timeout} +} + +func (c *client) do(ctx context.Context, method, path, token, key string, body []byte, basic string) result { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return result{err: err} + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + switch { + case basic != "": + req.Header.Set("Authorization", "Basic "+basic) + case token != "": + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + start := time.Now() + resp, err := c.http.Do(req) + if err != nil { + return result{err: err, latency: time.Since(start)} + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + r := result{status: resp.StatusCode, instance: resp.Header.Get("X-Instance-Id"), body: payload, latency: time.Since(start), err: err} + return r +} + +func (c *client) postJSON(ctx context.Context, path, token, key string, body any) result { + raw, err := json.Marshal(body) + if err != nil { + return result{err: err} + } + return c.do(ctx, http.MethodPost, path, token, key, raw, "") +} + +// expect decodes the body of a result with the wanted status. +func expect(r result, want int, out any) error { + if r.err != nil { + return r.err + } + if r.status != want { + return fmt.Errorf("status %d, want %d: %s", r.status, want, r.body) + } + if out == nil { + return nil + } + return json.Unmarshal(r.body, out) +} + +func (c *client) register(ctx context.Context, username, password string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]string{"username": username, "display_name": username, "password": password} + err := expect(c.postJSON(ctx, "/api/v1/users", "", "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) login(ctx context.Context, username, password string) (string, error) { + var session struct { + Token string `json:"token"` + } + basic := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + err := expect(c.do(ctx, http.MethodPost, "/api/v1/auth/sessions", "", "", nil, basic), http.StatusCreated, &session) + return session.Token, err +} + +func (c *client) createChat(ctx context.Context, token, title string, members []string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]any{"title": title, "member_ids": members} + err := expect(c.postJSON(ctx, "/api/v1/chats", token, "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) send(ctx context.Context, chatID, token, key, text string) result { + return c.postJSON(ctx, "/api/v1/chats/"+chatID+"/messages", token, key, map[string]string{"text": text}) +} + +// firstPage reads the oldest page of a history: the read of the profile. +func (c *client) firstPage(ctx context.Context, chatID, token string) result { + return c.do(ctx, http.MethodGet, "/api/v1/chats/"+chatID+"/messages?limit=50", token, "", nil, "") +} + +type message struct { + ID string `json:"id"` + Text string `json:"text"` +} + +// history reads a whole chat, page by page. +func (c *client) history(ctx context.Context, chatID, token string) ([]message, error) { + var all []message + path := "/api/v1/chats/" + chatID + "/messages?limit=100" + for range 100000 { + var page struct { + Items []message `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if err := expect(c.do(ctx, http.MethodGet, path, token, "", nil, ""), http.StatusOK, &page); err != nil { + return nil, err + } + all = append(all, page.Items...) + if page.NextCursor == nil { + return all, nil + } + path = "/api/v1/chats/" + chatID + "/messages?limit=100&cursor=" + url.QueryEscape(*page.NextCursor) + } + return nil, fmt.Errorf("history of %s did not end", chatID) +} diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go new file mode 100644 index 00000000..2873bfee --- /dev/null +++ b/tools/loadgen/main.go @@ -0,0 +1,343 @@ +// Command loadgen is the open-loop load generator of the experiments. +// +// loadgen bench — seeds data, warms up, then offers each rate of -rates +// for -step as a mix of history reads and message sends. +// loadgen timeline — one user sends messages at -rate, retrying uncertain +// sends with the same Idempotency-Key, then reads the +// history back: every confirmed message once. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + mrand "math/rand/v2" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" +) + +type config struct { + Mode string `json:"mode"` + BaseURL string `json:"base_url"` + Users int `json:"users"` + Chats int `json:"chats"` + SeedMessages int `json:"seed_messages_per_chat"` + ReadRatio float64 `json:"read_ratio"` + Rates []float64 `json:"rates"` + Step string `json:"step"` + Warmup string `json:"warmup"` + Rate float64 `json:"rate"` + Duration string `json:"duration"` + Timeout string `json:"client_timeout"` + Seed uint64 `json:"seed"` + Scrape bool `json:"scrape"` + StopOnFail bool `json:"stop_on_fail"` + StatsURL string `json:"stats_url"` + APIHost string `json:"api_host"` + DebugPort string `json:"debug_port"` + Out string `json:"-"` + + step, warmup, duration, timeout time.Duration +} + +func main() { + if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") { + fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags") + os.Exit(2) + } + cfg, err := parse(os.Args[1], os.Args[2:]) + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if cfg.Mode == "bench" { + err = runBench(ctx, cfg) + } else { + err = runTimeline(ctx, cfg) + } + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(1) + } +} + +func parse(mode string, args []string) (config, error) { + cfg := config{Mode: mode} + fs := flag.NewFlagSet(mode, flag.ContinueOnError) + rates := fs.String("rates", "200,400", "bench: offered requests per second of each step") + fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API") + fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users") + fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members") + fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring") + fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends") + fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step") + fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate") + fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second") + fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") + fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") + fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99") + fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") + fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") + fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") + fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)") + fs.StringVar(&cfg.Out, "out", "/out", "directory for the results") + if err := fs.Parse(args); err != nil { + return cfg, err + } + for _, part := range strings.Split(*rates, ",") { + r, err := strconv.ParseFloat(strings.TrimSpace(part), 64) + if err != nil || r <= 0 { + return cfg, fmt.Errorf("rates: %q is not a positive number", part) + } + cfg.Rates = append(cfg.Rates, r) + } + for _, d := range []struct { + name string + raw string + dst *time.Duration + }{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} { + v, err := time.ParseDuration(d.raw) + if err != nil || v < 0 { + return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw) + } + *d.dst = v + } + if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 { + return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive") + } + return cfg, os.MkdirAll(cfg.Out, 0o755) +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0o644) +} + +// startScrape returns the function that stops the sampling and waits. +func startScrape(ctx context.Context, cfg config, start time.Time) func() { + if !cfg.Scrape { + return func() {} + } + ctx, cancel := context.WithCancel(ctx) + s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}} + var wg sync.WaitGroup + wg.Go(func() { + if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil { + fmt.Fprintln(os.Stderr, "loadgen: scrape:", err) + } + }) + return func() { cancel(); wg.Wait() } +} + +type benchReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Users int `json:"users"` + Chats int `json:"chats"` + Seeded int `json:"seeded_messages"` + Steps []stepSummary `json:"steps"` + WritesOK int `json:"messages_written_during_steps"` +} + +func runBench(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages) + if err != nil { + return err + } + report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages} + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + + // next runs on the scheduling goroutine, which owns the random source. + counter := 0 + next := func(rec *recorder) func(time.Time) func() { + return func(planned time.Time) func() { + r := ds.rooms[rng.IntN(len(ds.rooms))] + a := ds.actors[r.members[rng.IntN(len(r.members))]] + write := rng.Float64() >= cfg.ReadRatio + counter++ + text := fmt.Sprintf("load %d", counter) + return func() { + sent := time.Now() + var res result + if write { + res = c.send(ctx, r.id, a.token, uuid.NewString(), text) + } else { + res = c.firstPage(ctx, r.id, a.token) + } + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write}) + } + } + } + if cfg.warmup > 0 { + openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{})) + } + all := &recorder{} + for _, rate := range cfg.Rates { + if ctx.Err() != nil { + break + } + stepStart := time.Now() + rec := &recorder{} + openLoop(ctx, rate, cfg.step, next(rec)) + s := summarize(rate, cfg.step, rec.all()) + s.StartedAt = stepStart.Sub(start).Seconds() + report.Steps = append(report.Steps, s) + report.WritesOK += s.Writes + for _, x := range rec.all() { + all.add(x) + } + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) + if cfg.StopOnFail && s.overloaded() { + break + } + } + stopScrape() + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { + return err + } + return writeJSON(filepath.Join(cfg.Out, "steps.json"), report) +} + +type timelineReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Jobs int `json:"messages_attempted"` + Attempts int `json:"requests"` + Confirmed int `json:"confirmed"` + RetriedJobs int `json:"messages_retried"` + Unresolved int `json:"unresolved"` + Rejected int `json:"rejected_4xx"` + Classes map[string]int `json:"classes"` + Instances map[string]int `json:"instances"` + History int `json:"history_messages"` + Duplicates int `json:"duplicates"` + Missing int `json:"missing_confirmed"` + Passed bool `json:"passed"` +} + +// retryDelay spaces the attempts of one message whose outcome was uncertain. +const retryDelay = 200 * time.Millisecond + +// retryGrace bounds the retries of an uncertain message after the schedule ends. +const retryGrace = 10 * time.Second + +func runTimeline(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0) + if err != nil { + return err + } + r := ds.rooms[0] + sender := ds.actors[r.members[0]] + report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}} + runID := randomHex(3) + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + deadline := start.Add(cfg.duration + retryGrace) + + var mu sync.Mutex + confirmed := map[string]string{} // text -> message id + rec := &recorder{} + jobs := 0 + openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() { + jobs++ + key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs) + return func() { + for attempt := 0; ; attempt++ { + sent := time.Now() + res := c.send(ctx, r.id, sender.token, key, text) + cls := classify(res) + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true}) + mu.Lock() + switch { + case cls == "ok": + var m message + if json.Unmarshal(res.body, &m) == nil { + confirmed[text] = m.ID + } + case res.err == nil && res.status >= 400 && res.status < 500: + report.Rejected++ + case time.Now().After(deadline) || ctx.Err() != nil: + report.Unresolved++ + default: + if attempt == 0 { + report.RetriedJobs++ + } + mu.Unlock() + time.Sleep(retryDelay) + planned = time.Now() + continue + } + mu.Unlock() + return + } + } + }) + stopScrape() + samples := rec.all() + report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed) + for _, x := range samples { + report.Classes[x.class]++ + if x.instance != "" { + report.Instances[x.instance]++ + } + } + + history, err := c.history(ctx, r.id, sender.token) + if err != nil { + return fmt.Errorf("read history back: %w", err) + } + report.History = len(history) + seen := map[string]int{} + byID := map[string]string{} + for _, m := range history { + seen[m.Text]++ + byID[m.ID] = m.Text + } + for _, n := range seen { + if n > 1 { + report.Duplicates += n - 1 + } + } + for text, id := range confirmed { + if byID[id] != text { + report.Missing++ + } + } + report.Passed = report.Duplicates == 0 && report.Missing == 0 + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil { + return err + } + if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil { + return err + } + fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n", + report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing) + if !report.Passed { + return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing) + } + return nil +} diff --git a/tools/loadgen/main_test.go b/tools/loadgen/main_test.go new file mode 100644 index 00000000..4f6c8c75 --- /dev/null +++ b/tools/loadgen/main_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func liveServer(t *testing.T) string { + return serve(t, nil) +} + +func serve(t *testing.T, wrap func(http.Handler) http.Handler) string { + t.Helper() + a := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "loadgen-test", + }) + var h http.Handler = a.Engine + if wrap != nil { + h = wrap(h) + } + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return srv.URL +} + +// Registration hashes a password with Argon2id, which is slow on a small +// replica; the setup is not measured, so it must not fail on the short +// timeout of the measured requests. +func TestSlowSetupDoesNotFailTheRun(t *testing.T) { + slowAccounts := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && (r.URL.Path == "/api/v1/users" || r.URL.Path == "/api/v1/auth/sessions") { + time.Sleep(300 * time.Millisecond) + } + next.ServeHTTP(w, r) + }) + } + cfg, err := parse("bench", []string{"-base-url", serve(t, slowAccounts), "-users", "2", "-chats", "1", "-seed-messages", "1", + "-rates", "10", "-step", "1s", "-warmup", "0s", "-timeout", "100ms", "-out", t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatalf("a setup slower than -timeout failed the run: %v", err) + } +} + +func TestBenchAgainstLiveServer(t *testing.T) { + out := t.TempDir() + cfg, err := parse("bench", []string{"-base-url", liveServer(t), "-users", "4", "-chats", "2", "-seed-messages", "3", + "-rates", "20,40", "-step", "1s", "-warmup", "0s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report benchReport + raw, err := os.ReadFile(filepath.Join(out, "steps.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("steps.json: %v %s", err, raw) + } + if len(report.Steps) != 2 || report.Seeded != 6 || report.Users != 4 || report.Chats != 2 { + t.Fatalf("report = %+v", report) + } + for _, s := range report.Steps { + if s.Sent != int(s.OfferedRPS) || s.OK != s.Sent || s.ErrorShare != 0 || s.Instances["loadgen-test"] != s.Sent { + t.Fatalf("step = %+v", s) + } + } + csv, err := os.ReadFile(filepath.Join(out, "seconds.csv")) + if err != nil || !strings.HasPrefix(string(csv), "second,sent,ok,") || strings.Count(string(csv), "\n") < 3 { + t.Fatalf("seconds.csv: %v\n%s", err, csv) + } +} + +func TestTimelineVerifiesHistory(t *testing.T) { + out := t.TempDir() + cfg, err := parse("timeline", []string{"-base-url", liveServer(t), "-rate", "20", "-duration", "1s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runTimeline(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report timelineReport + raw, err := os.ReadFile(filepath.Join(out, "timeline.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("timeline.json: %v %s", err, raw) + } + if report.Jobs != 20 || report.Confirmed != 20 || report.History != 20 || report.Duplicates != 0 || report.Missing != 0 || !report.Passed { + t.Fatalf("report = %+v", report) + } +} diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go new file mode 100644 index 00000000..7a84f6a2 --- /dev/null +++ b/tools/loadgen/run.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "sync" + "time" +) + +// openLoop sends request i at start + i/rate whatever the response times: a +// closed loop would let a slow system lower its own load and hide its latency +// (coordinated omission). It returns once every job has finished. +func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { + n := int(rate * d.Seconds()) + interval := time.Duration(float64(time.Second) / rate) + start := time.Now() + var wg sync.WaitGroup + for i := range n { + planned := start.Add(time.Duration(i) * interval) + if wait := time.Until(planned); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + if ctx.Err() != nil { + break + } + wg.Go(next(planned)) + } + wg.Wait() +} + +// recorder collects samples from concurrent jobs. +type recorder struct { + mu sync.Mutex + samples []sample +} + +func (r *recorder) add(s sample) { + r.mu.Lock() + r.samples = append(r.samples, s) + r.mu.Unlock() +} + +func (r *recorder) all() []sample { + r.mu.Lock() + defer r.mu.Unlock() + return append([]sample(nil), r.samples...) +} diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go new file mode 100644 index 00000000..6b95db75 --- /dev/null +++ b/tools/loadgen/scrape.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "io" + "net" + "net/http" + "os" + "strings" + "time" +) + +// scraper samples the HAProxy statistics and every replica's expvar once a +// second; replicas are found through Docker DNS, as HAProxy finds them. +type scraper struct { + statsURL string + apiHost string + debugPort string + http *http.Client +} + +type scrapeLine struct { + T float64 `json:"t"` + HAProxy []map[string]string `json:"haproxy,omitempty"` + Replicas []json.RawMessage `json:"replicas,omitempty"` +} + +// haproxyFields are the columns of the statistics CSV that are kept. +var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"} + +func (s scraper) run(ctx context.Context, start time.Time, path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)} + if err := enc.Encode(line); err != nil { + return err + } + } +} + +func (s scraper) get(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := s.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) +} + +// haproxy keeps the rows of the replicas backend. +func (s scraper) haproxy(ctx context.Context) []map[string]string { + raw, err := s.get(ctx, s.statsURL) + if err != nil { + return nil + } + rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll() + if err != nil || len(rows) < 2 { + return nil + } + header := rows[0] + col := map[string]int{} + for i, name := range header { + col[name] = i + } + var out []map[string]string + for _, row := range rows[1:] { + if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") { + continue + } + kept := map[string]string{} + for _, name := range haproxyFields { + if i, ok := col[name]; ok { + kept[name] = row[i] + } + } + out = append(out, kept) + } + return out +} + +// replicas returns the expvar documents of the running replicas, trimmed to +// the fields the study uses. +func (s scraper) replicas(ctx context.Context) []json.RawMessage { + addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost) + if err != nil { + return nil + } + var out []json.RawMessage + for _, addr := range addrs { + raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars") + if err != nil { + continue + } + var vars struct { + Instance string `json:"instance"` + GOMAXPROCS int `json:"gomaxprocs"` + Pool json.RawMessage `json:"pgxpool"` + Mem struct { + HeapAlloc uint64 `json:"HeapAlloc"` + Sys uint64 `json:"Sys"` + NumGC uint32 `json:"NumGC"` + PauseTotalNs uint64 `json:"PauseTotalNs"` + GCCPUFraction float64 `json:"GCCPUFraction"` + } `json:"memstats"` + } + if json.Unmarshal(raw, &vars) != nil { + continue + } + kept, err := json.Marshal(map[string]any{ + "addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS, + "pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys, + "num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction, + }) + if err == nil { + out = append(out, kept) + } + } + return out +} diff --git a/tools/loadgen/seed.go b/tools/loadgen/seed.go new file mode 100644 index 00000000..61624236 --- /dev/null +++ b/tools/loadgen/seed.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + mrand "math/rand/v2" + "sync" + "time" + + "github.com/google/uuid" +) + +// seedPassword is shared by the synthetic users of one run. +const seedPassword = "loadgen-synthetic-password" + +// seedTimeout bounds one setup request. The setup is not measured, and +// registration hashes a password with Argon2id, slow on a small replica. +const seedTimeout = 30 * time.Second + +// seedWorkers bounds the parallel registrations: each one hashes a password +// with Argon2id, and the setup must not be what the study measures. +const seedWorkers = 8 + +type actor struct { + id, token string +} + +type room struct { + id string + members []int // indexes into dataset.actors +} + +type dataset struct { + actors []actor + rooms []room + // messages is the number of messages written while seeding. + messages int +} + +// seed registers users and logs each in once, creates chats of two to five +// random members and writes perChat messages into each. Logins happen here, +// so the password hashing never shows up in the measured steps. +func seed(ctx context.Context, c *client, rng *mrand.Rand, users, chats, perChat int) (dataset, error) { + if users < 2 || chats < 1 { + return dataset{}, fmt.Errorf("need at least 2 users and 1 chat") + } + runID := randomHex(3) + ds := dataset{actors: make([]actor, users)} + if err := parallel(users, func(i int) error { + name := fmt.Sprintf("lg_%s_%d", runID, i) + id, err := c.register(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("register %s: %w", name, err) + } + token, err := c.login(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("log in %s: %w", name, err) + } + ds.actors[i] = actor{id: id, token: token} + return nil + }); err != nil { + return dataset{}, err + } + + for j := range chats { + size := 2 + rng.IntN(4) + members := rng.Perm(users)[:min(size, users)] + ids := make([]string, len(members)) + for k, m := range members { + ids[k] = ds.actors[m].id + } + id, err := c.createChat(ctx, ds.actors[members[0]].token, fmt.Sprintf("load %s %d", runID, j), ids) + if err != nil { + return dataset{}, fmt.Errorf("create chat %d: %w", j, err) + } + ds.rooms = append(ds.rooms, room{id: id, members: members}) + } + + if err := parallel(len(ds.rooms), func(j int) error { + r := ds.rooms[j] + for k := range perChat { + sender := ds.actors[r.members[k%len(r.members)]] + res := c.send(ctx, r.id, sender.token, uuid.NewString(), fmt.Sprintf("seed %d", k)) + if err := expect(res, 201, nil); err != nil { + return fmt.Errorf("seed message into chat %d: %w", j, err) + } + } + return nil + }); err != nil { + return dataset{}, err + } + ds.messages = len(ds.rooms) * perChat + return ds, nil +} + +// parallel runs fn(0..n-1) on seedWorkers goroutines and returns the first +// error. +func parallel(n int, fn func(i int) error) error { + var ( + wg sync.WaitGroup + mu sync.Mutex + first error + ) + next := make(chan int) + for range seedWorkers { + wg.Go(func() { + for i := range next { + if err := fn(i); err != nil { + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + } + } + }) + } + for i := range n { + next <- i + } + close(next) + wg.Wait() + return first +} + +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go new file mode 100644 index 00000000..8f5dd8ca --- /dev/null +++ b/tools/loadgen/stats.go @@ -0,0 +1,205 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "slices" + "sort" + "strings" + "time" +) + +// maxValidLag: if requests left later than this at p99, the generator, not +// the system, limited the step. +const maxValidLag = 10 * time.Millisecond + +// The capacity criterion of the study: a step passes when it is valid, has +// fewer than 1 % errors and its successful requests have a p99 within 500 ms. +const ( + maxErrorShare = 0.01 + sloP99 = 500 * time.Millisecond +) + +// sample is one request as the generator saw it. +type sample struct { + planned time.Time + lag time.Duration // actual send time minus planned send time + latency time.Duration + class string + instance string + write bool +} + +// classify names the outcome of a request. Errors of the balancer itself carry +// no X-Instance-Id and get the suffix _lb; client_timeout is the generator +// giving up, not a timeout of the system. +func classify(r result) string { + switch { + case r.err != nil && isTimeout(r.err): + return "client_timeout" + case r.err != nil: + return "conn_error" + case r.status >= 200 && r.status < 300: + return "ok" + case r.instance == "": + return fmt.Sprintf("http_%d_lb", r.status) + default: + return fmt.Sprintf("http_%d", r.status) + } +} + +func isTimeout(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// percentile is the nearest-rank percentile of an ascending slice. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + rank := int(math.Ceil(p / 100 * float64(len(sorted)))) + return sorted[min(max(rank, 1), len(sorted))-1] +} + +type stepSummary struct { + OfferedRPS float64 `json:"offered_rps"` + Seconds float64 `json:"seconds"` + StartedAt float64 `json:"started_at_s"` + Sent int `json:"sent"` + OK int `json:"ok"` + OKRPS float64 `json:"ok_rps"` + ErrorShare float64 `json:"error_share"` + Classes map[string]int `json:"classes"` + P50Ms float64 `json:"ok_p50_ms"` + P95Ms float64 `json:"ok_p95_ms"` + P99Ms float64 `json:"ok_p99_ms"` + LagP99Ms float64 `json:"lag_p99_ms"` + Valid bool `json:"valid"` + Passed bool `json:"passed"` + Instances map[string]int `json:"instances"` + Writes int `json:"writes_ok"` +} + +// summarize reduces one step; latency percentiles cover successful requests. +func summarize(rate float64, d time.Duration, samples []sample) stepSummary { + s := stepSummary{ + OfferedRPS: rate, + Seconds: d.Seconds(), + Sent: len(samples), + Classes: map[string]int{}, + Instances: map[string]int{}, + } + var ok, lags []time.Duration + for _, x := range samples { + s.Classes[x.class]++ + lags = append(lags, x.lag) + if x.instance != "" { + s.Instances[x.instance]++ + } + if x.class == "ok" { + ok = append(ok, x.latency) + if x.write { + s.Writes++ + } + } + } + slices.Sort(ok) + slices.Sort(lags) + s.OK = len(ok) + s.OKRPS = float64(s.OK) / d.Seconds() + if s.Sent > 0 { + s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent) + } + s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)) + lagP99 := percentile(lags, 99) + s.LagP99Ms = ms(lagP99) + s.Valid = lagP99 <= maxValidLag + s.Passed = s.Valid && !s.overloaded() + return s +} + +func (s stepSummary) overloaded() bool { + return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99) +} + +func ms(d time.Duration) float64 { + return float64(d.Microseconds()) / 1000 +} + +// errorColumns are the fixed error columns of the per-second CSV; any other +// class lands in err_other. +var errorColumns = []string{"client_timeout", "conn_error"} + +// secondsCSV renders the per-second timeline, attributed by planned send +// time so it lines up with the moment a fault was injected. +func secondsCSV(start time.Time, samples []sample) string { + type bucket struct { + samples []sample + } + buckets := map[int]*bucket{} + last := 0 + for _, x := range samples { + sec := int(x.planned.Sub(start) / time.Second) + if buckets[sec] == nil { + buckets[sec] = &bucket{} + } + buckets[sec].samples = append(buckets[sec].samples, x) + last = max(last, sec) + } + var b strings.Builder + b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n") + for sec := 0; sec <= last; sec++ { + bk := buckets[sec] + if bk == nil { + fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec) + continue + } + var ok, lags []time.Duration + counts := map[string]int{} + instances := map[string]int{} + for _, x := range bk.samples { + lags = append(lags, x.lag) + if x.instance != "" { + instances[x.instance]++ + } + switch { + case x.class == "ok": + ok = append(ok, x.latency) + case slices.Contains(errorColumns, x.class): + counts[x.class]++ + case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"): + counts["http_5xx_lb"]++ + case strings.HasPrefix(x.class, "http_5"): + counts["http_5xx_api"]++ + case strings.HasPrefix(x.class, "http_4"): + counts["http_4xx"]++ + default: + counts["err_other"]++ + } + } + slices.Sort(ok) + slices.Sort(lags) + names := make([]string, 0, len(instances)) + for name := range instances { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, name := range names { + parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name])) + } + fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n", + sec, len(bk.samples), len(ok), + counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"], + ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)), + strings.Join(parts, "|")) + } + return b.String() +} diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go new file mode 100644 index 00000000..f9a8bf1b --- /dev/null +++ b/tools/loadgen/stats_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" +) + +type timeoutErr struct{} + +func (timeoutErr) Error() string { return "i/o timeout" } +func (timeoutErr) Timeout() bool { return true } +func (timeoutErr) Temporary() bool { return true } + +var _ net.Error = timeoutErr{} + +func TestClassify(t *testing.T) { + for _, c := range []struct { + res result + want string + }{ + {result{status: 201, instance: "a"}, "ok"}, + {result{status: 200, instance: "a"}, "ok"}, + {result{status: 409, instance: "a"}, "http_409"}, + {result{status: 503, instance: "a"}, "http_503"}, + {result{status: 503}, "http_503_lb"}, + {result{status: 502}, "http_502_lb"}, + {result{err: context.DeadlineExceeded}, "client_timeout"}, + {result{err: fmt.Errorf("post: %w", timeoutErr{})}, "client_timeout"}, + {result{err: errors.New("connection refused")}, "conn_error"}, + } { + if got := classify(c.res); got != c.want { + t.Errorf("classify(%+v) = %q, want %q", c.res, got, c.want) + } + } +} + +func TestPercentileNearestRank(t *testing.T) { + ms := func(v ...int) []time.Duration { + out := make([]time.Duration, len(v)) + for i, x := range v { + out[i] = time.Duration(x) * time.Millisecond + } + return out + } + sorted := ms(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + for p, want := range map[float64]time.Duration{50: 5 * time.Millisecond, 95: 10 * time.Millisecond, 99: 10 * time.Millisecond, 10: time.Millisecond} { + if got := percentile(sorted, p); got != want { + t.Errorf("p%v = %v, want %v", p, got, want) + } + } + if got := percentile(nil, 99); got != 0 { + t.Errorf("empty p99 = %v, want 0", got) + } +} + +// A step is invalid when the generator itself fell behind its schedule: the +// system was then offered less than the nominal rate. +func TestStepSummaryValidityAndErrorShare(t *testing.T) { + samples := []sample{ + {class: "ok", latency: 10 * time.Millisecond, lag: time.Millisecond}, + {class: "ok", latency: 20 * time.Millisecond, lag: time.Millisecond}, + {class: "client_timeout", latency: 2 * time.Second, lag: time.Millisecond}, + {class: "http_503_lb", latency: time.Millisecond, lag: time.Millisecond}, + } + s := summarize(4, time.Second, samples) + if s.Sent != 4 || s.OK != 2 || s.OKRPS != 2 || s.ErrorShare != 0.5 || !s.Valid { + t.Fatalf("summary = %+v", s) + } + if s.Classes["client_timeout"] != 1 || s.Classes["http_503_lb"] != 1 { + t.Fatalf("classes = %v", s.Classes) + } + if s.P99Ms != 20 { + t.Fatalf("p99 of successful requests = %v ms, want 20", s.P99Ms) + } + samples[0].lag = 50 * time.Millisecond + samples[1].lag = 50 * time.Millisecond + if s := summarize(4, time.Second, samples); s.Valid { + t.Fatalf("lagging generator marked valid: %+v", s) + } +} + +// Open loop: requests leave on schedule even when responses are slow. A +// closed loop with one worker would send only a handful here. +func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { + var fired atomic.Int32 + start := time.Now() + openLoop(context.Background(), 50, time.Second, func(time.Time) func() { + return func() { + fired.Add(1) + time.Sleep(200 * time.Millisecond) + } + }) + if n := fired.Load(); n < 45 || n > 50 { + t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) + } + // About 1.2 s; the bound leaves room for -race on a shared CI runner and + // still fails a closed loop, which would need 50 x 200 ms = 10 s. + if took := time.Since(start); took > 2500*time.Millisecond { + t.Fatalf("open loop took %v, want about 1.2 s", took) + } +} + +func TestStepPassesOnlyWithinTheObjective(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, fast); !s.Passed { + t.Fatalf("fast clean step must pass: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); s.Passed { + t.Fatalf("p99 above 500 ms must fail: %+v", s) + } + failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond}) + if s := summarize(3, time.Second, failing); s.Passed { + t.Fatalf("errors above 1 %% must fail: %+v", s) + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.Passed { + t.Fatalf("an invalid step must fail: %+v", s) + } +} + +// The ladder stops when the system leaves the objective, not when the +// generator alone fell behind: a later step may still be valid and pass. +func TestOnlyTheSystemEndsTheLadder(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.overloaded() { + t.Fatalf("a lagging generator must not end the ladder: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); !s.overloaded() { + t.Fatalf("p99 above 500 ms must end the ladder: %+v", s) + } + failing := []sample{ok(10*time.Millisecond, time.Millisecond), {class: "client_timeout", lag: time.Millisecond}} + if s := summarize(2, time.Second, failing); !s.overloaded() { + t.Fatalf("errors above 1 %% must end the ladder: %+v", s) + } +} -- 2.54.0 From d761861def80096fe0897a3b693eda6ff8d0c0c7 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:36:59 +0300 Subject: [PATCH 2/6] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D1=8C=D1=82?= =?UTF-8?q?=D0=B5=20S3-=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D1=81=20=D0=B4=D0=B5=D0=B4=D1=83=D0=BF=D0=BB=D0=B8=D0=BA=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D0=B5=D0=B9=20=D0=B8=20=D0=B1=D0=B5=D0=B7=D0=BE?= =?UTF-8?q?=D0=BF=D0=B0=D1=81=D0=BD=D0=BE=D0=B9=20=D1=81=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=BE=D0=B9=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8?= =?UTF-8?q?=D1=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- Dockerfile | 4 +- cmd/reconcile/main.go | 99 +++ cmd/s3init/main.go | 30 + cmd/server/main.go | 46 ++ compose.yaml | 137 +++- go.mod | 25 +- go.sum | 36 +- infra/haproxy/haproxy.cfg | 2 + infra/s3/README.md | 77 ++ infra/s3/seaweedfs.options | 31 + internal/app/app.go | 8 + internal/app/app_test.go | 5 +- internal/app/attachment_test.go | 171 +++++ internal/attachment/attachment.go | 22 + internal/attachment/blob.go | 25 + internal/attachment/handler.go | 121 +++ internal/attachment/lock.go | 23 + internal/attachment/reconcile.go | 479 ++++++++++++ internal/attachment/reconcile_test.go | 708 ++++++++++++++++++ internal/attachment/schema_test.go | 27 + internal/attachment/service.go | 172 +++++ internal/attachment/service_test.go | 270 +++++++ internal/attachment/store/postgres.go | 79 ++ internal/attachment/validation.go | 51 ++ internal/attachment/validation_test.go | 93 +++ internal/chat/attachment_handler_test.go | 72 ++ internal/chat/attachment_http_test.go | 125 ++++ internal/chat/attachment_message_test.go | 66 ++ internal/chat/handler.go | 68 +- internal/chat/message.go | 49 +- internal/chat/service.go | 5 +- internal/chat/service_test.go | 5 +- internal/chat/store/attachments_test.go | 312 ++++++++ internal/chat/store/postgres.go | 128 +++- internal/chat/store/postgres_test.go | 3 +- internal/platform/config/config.go | 11 +- internal/platform/config/config_test.go | 15 +- internal/platform/config/s3.go | 55 ++ internal/platform/config/s3_test.go | 27 + internal/platform/httpx/middleware.go | 5 + internal/platform/s3store/bootstrap_test.go | 142 ++++ internal/platform/s3store/integration_test.go | 270 +++++++ internal/platform/s3store/readiness.go | 114 +++ internal/platform/s3store/readiness_test.go | 48 ++ internal/platform/s3store/store.go | 198 +++++ internal/platform/s3store/store_test.go | 171 +++++ internal/server/server.go | 4 +- internal/server/server_test.go | 177 +++++ internal/testenv/s3.go | 206 +++++ migrations/00003_attachments.sql | 35 + scripts/init-secrets.sh | 80 +- scripts/restart.sh | 5 + scripts/wait-ready.sh | 22 + 54 files changed, 5061 insertions(+), 102 deletions(-) create mode 100644 cmd/reconcile/main.go create mode 100644 cmd/s3init/main.go create mode 100644 infra/s3/README.md create mode 100644 infra/s3/seaweedfs.options create mode 100644 internal/app/attachment_test.go create mode 100644 internal/attachment/attachment.go create mode 100644 internal/attachment/blob.go create mode 100644 internal/attachment/handler.go create mode 100644 internal/attachment/lock.go create mode 100644 internal/attachment/reconcile.go create mode 100644 internal/attachment/reconcile_test.go create mode 100644 internal/attachment/schema_test.go create mode 100644 internal/attachment/service.go create mode 100644 internal/attachment/service_test.go create mode 100644 internal/attachment/store/postgres.go create mode 100644 internal/attachment/validation.go create mode 100644 internal/attachment/validation_test.go create mode 100644 internal/chat/attachment_handler_test.go create mode 100644 internal/chat/attachment_http_test.go create mode 100644 internal/chat/attachment_message_test.go create mode 100644 internal/chat/store/attachments_test.go create mode 100644 internal/platform/config/s3.go create mode 100644 internal/platform/config/s3_test.go create mode 100644 internal/platform/s3store/bootstrap_test.go create mode 100644 internal/platform/s3store/integration_test.go create mode 100644 internal/platform/s3store/readiness.go create mode 100644 internal/platform/s3store/readiness_test.go create mode 100644 internal/platform/s3store/store.go create mode 100644 internal/platform/s3store/store_test.go create mode 100644 internal/testenv/s3.go create mode 100644 migrations/00003_attachments.sql create mode 100644 scripts/wait-ready.sh diff --git a/.env.example b/.env.example index d99d26a2..85dc5bfe 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,12 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_RECONCILE_ACCESS_KEY_ID_FILE=/run/secrets/s3_reconcile_access_key_id +S3_RECONCILE_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_reconcile_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/Dockerfile b/Dockerfile index 2a0b1902..a3e94d88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY cmd ./cmd COPY internal ./internal COPY migrations ./migrations RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init # The load generator gets an image of its own; the API image does not carry it. FROM build AS loadgen-build @@ -25,7 +25,7 @@ ENTRYPOINT ["/usr/local/bin/loadgen"] # Last, so that `build: .` builds it; it also runs the migrate service. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app -COPY --from=build /out/server /out/migrate /usr/local/bin/ +COPY --from=build /out/server /out/migrate /out/reconcile /out/s3init /usr/local/bin/ USER app ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 00000000..b7090a83 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,99 @@ +// Command reconcile reports attachment inconsistencies as JSON lines. Mutations +// require --apply; --deep also streams referenced objects to verify SHA256. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "messenger/internal/attachment" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, out, errOut io.Writer) int { + flags := flag.NewFlagSet("reconcile", flag.ContinueOnError) + flags.SetOutput(errOut) + apply := flags.Bool("apply", false, "apply repairs (default: read-only dry run)") + deep := flags.Bool("deep", false, "stream referenced objects and verify SHA256 and size") + grace := flags.Duration("grace", time.Hour, "minimum file/object age; must be at least 2m") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() != 0 || *grace < 2*time.Minute { + fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m") + return 2 + } + dbConfig, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + // Compose mounts the reconciler identity at the standard credential paths. + s3Config, err := config.LoadS3(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 1 + } + defer db.Close() + r, err := attachment.NewReconciler(db, blobs, *grace) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 2 + } + // A broken output stream cancels the scan so repairs cannot continue after + // the caller loses the audit trail. Print partial statistics on scan errors. + scanCtx, cancel := context.WithCancel(ctx) + defer cancel() + encoder := json.NewEncoder(out) + var outputErr error + stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) { + if outputErr == nil { + outputErr = encoder.Encode(f) + if outputErr != nil { + cancel() + } + } + }) + if outputErr == nil { + outputErr = encoder.Encode(stats) + } + if outputErr != nil { + fmt.Fprintln(errOut, "reconcile output:", outputErr) + return 1 + } + if runErr != nil { + fmt.Fprintln(errOut, "reconcile:", runErr) + return 1 + } + return 0 +} diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go new file mode 100644 index 00000000..db308002 --- /dev/null +++ b/cmd/s3init/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "fmt" + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" + "os" + "time" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "bucket bootstrap failed:", err) + os.Exit(1) + } +} +func run() error { + cfg, err := config.LoadS3(os.LookupEnv) + if err != nil { + return err + } + s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return s.Bootstrap(ctx) +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 9d4f4b94..cb4aec4a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,10 +17,12 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "messenger/internal/app" + "messenger/internal/attachment" "messenger/internal/chat" "messenger/internal/platform/config" "messenger/internal/platform/ident" "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" "messenger/internal/platform/valkey" "messenger/internal/server" ) @@ -59,6 +61,26 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } defer sessions.Close() + var blobs attachment.Blobs + var reconciler *attachment.Reconciler + if cfg.S3.Endpoint != "" { + blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey) + if err != nil { + return err + } + reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv)) + if e != nil { + return e + } + reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey) + if e != nil { + return e + } + reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour) + if err != nil { + return err + } + } ln, err := net.Listen("tcp", cfg.Addr()) if err != nil { @@ -77,6 +99,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) MaxBodyBytes: cfg.MaxBodyBytes, InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, + Blobs: blobs, + MaxUploadBytes: cfg.S3.MaxBytes, + MaxImagePixels: cfg.S3.MaxPixels, }) var debugLn net.Listener if cfg.DebugAddr != "" { @@ -89,6 +114,27 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) ctx, cancel := context.WithCancel(ctx) var background sync.WaitGroup background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if reconciler != nil { + background.Go(func() { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + scan, cancel := context.WithTimeout(ctx, 2*time.Minute) + stats, err := reconciler.Run(scan, true, false, nil) + cancel() + if err != nil { + log.Warn("attachment cleanup incomplete", "error", err) + } else { + log.Info("attachment cleanup", "scanned_files", stats.ScannedFiles, "repaired", stats.Repaired) + } + } + } + }) + } if debugLn != nil { background.Go(func() { if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { diff --git a/compose.yaml b/compose.yaml index 5b3e7b7e..bc08ab33 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,5 @@ # Local stand: HAProxy in front of interchangeable API replicas over shared -# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data +# PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data # network has no way out. The load-test knobs are in compose.bench.yaml. name: lab4 @@ -38,11 +38,24 @@ services: SESSION_STORE_URL_FILE: /run/secrets/session_store_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - # Go does not read the cgroup memory limit. - GOMEMLIMIT: 230MiB + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + # The background reconciler uses a separate identity with List rights. + S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id + S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key + # One decoder per replica. Progressive JPEG retains coefficient buffers; + # a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer. + GOMEMLIMIT: 1536MiB secrets: - database_url - session_store_url + - s3_access_key_id + - s3_secret_access_key + - s3_reconcile_access_key_id + - s3_reconcile_secret_access_key networks: - edge - data @@ -53,14 +66,16 @@ services: condition: service_healthy valkey: condition: service_healthy - # Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL. - stop_grace_period: 15s + s3-init: + condition: service_completed_successfully + # Binary deadline 30s plus compensation, shutdown and drain margin. + stop_grace_period: 45s deploy: replicas: ${API_REPLICAS:-2} resources: limits: cpus: "1.0" - memory: 256M + memory: 2G migrate: image: mtusi-messenger:lab4 @@ -138,9 +153,105 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M + s3: + image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0 + command: ["server", "-options=/etc/seaweedfs/lab.options"] + environment: + GOMEMLIMIT: 460MiB + secrets: + - s3_identities + volumes: + - ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro + # Both volume data and the image's /data/filerldb2 metadata live here. + - s3data:/data + networks: + - data + healthcheck: + # A reachable anonymous request must be refused, including before init. + test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + start_interval: 1s + stop_grace_period: 30s + pids_limit: 256 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + pids: 256 + + s3-init: + build: . + image: mtusi-messenger:lab4 + entrypoint: ["/usr/local/bin/s3init"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - source: s3_admin_access_key_id + target: s3_access_key_id + - source: s3_admin_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + s3: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + reconcile: + profiles: ["recon"] + build: . + image: mtusi-messenger:lab4 + entrypoint: ["/usr/local/bin/reconcile"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - database_url + - source: s3_reconcile_access_key_id + target: s3_access_key_id + - source: s3_reconcile_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + volumes: pgdata: valkeydata: + s3data: networks: edge: {} @@ -156,3 +267,17 @@ secrets: file: ./secrets/postgres_password valkey_auth: file: ./secrets/valkey_auth.conf + s3_identities: + file: ./secrets/s3_identities.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_reconcile_access_key_id: + file: ./secrets/s3_reconcile_access_key_id + s3_reconcile_secret_access_key: + file: ./secrets/s3_reconcile_secret_access_key diff --git a/go.mod b/go.mod index 9c6fb13e..5439a0c6 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,20 @@ module messenger go 1.27.1 require ( + github.com/docker/docker v28.3.3+incompatible + github.com/docker/go-connections v0.6.0 github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 github.com/pressly/goose/v3 v3.27.0 github.com/redis/go-redis/v9 v9.23.0 github.com/testcontainers/testcontainers-go v0.39.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 golang.org/x/crypto v0.57.0 + google.golang.org/grpc v1.84.0 + google.golang.org/protobuf v1.36.12 ) require ( @@ -31,9 +36,8 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/creack/pty v1.1.24 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.3.3+incompatible // indirect - github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.1 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect @@ -54,12 +58,15 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect github.com/mfridman/interpolate v0.0.2 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect @@ -72,20 +79,24 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pelletier/go-toml/v2 v2.4.3 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sethvargo/go-retry v0.4.0 // indirect github.com/shirou/gopsutil/v4 v4.26.6 // indirect - github.com/sirupsen/logrus v1.9.4 // indirect + github.com/sirupsen/logrus v1.10.2 // indirect github.com/stretchr/testify v1.12.1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect @@ -97,11 +108,11 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.59.0 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/grpc v1.84.0 // indirect - google.golang.org/protobuf v1.36.12 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index fb12767b..9807f1b1 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -100,8 +102,11 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -116,6 +121,12 @@ github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5L github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -145,8 +156,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -162,12 +175,14 @@ github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBP github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -180,6 +195,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= @@ -188,6 +204,8 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwN github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g= github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= @@ -202,6 +220,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= @@ -249,8 +269,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -281,6 +301,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8= @@ -290,6 +312,8 @@ google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2Q google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 129badc3..50795f51 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -55,6 +55,8 @@ frontend entry default_backend replicas backend replicas + # Binary handlers have a 30s budget, ordinary API handlers retain 5s. + http-request set-timeout server 35s if { path_reg ^/api/v1/chats/[^/]+/attachments$|^/api/v1/attachments/[^/]+/content$ } balance roundrobin option httpchk http-check send meth GET uri /health/ready diff --git a/infra/s3/README.md b/infra/s3/README.md new file mode 100644 index 00000000..cc9c0bc5 --- /dev/null +++ b/infra/s3/README.md @@ -0,0 +1,77 @@ +# Private object storage + +`s3` runs SeaweedFS 4.53 in one process. The image is pinned by digest in +`compose.yaml` and `internal/testenv/s3.go`. `seaweedfs.options` is shared by +the stand and the disposable integration-test server. + +S3 HTTP (8333) and SeaweedFS's companion gRPC (18333) listen on the private +container network. This image has no flag to disable the companion listener; +its identity-cache mutations reject remote peers, which the integration test +checks. Master, volume and filer HTTP/gRPC endpoints bind to loopback. +No host port is published. `/data` +contains both object volumes and the image's LevelDB filer metadata; the +named `s3data` volume must be retained together with PostgreSQL data. + +Run `sh scripts/init-secrets.sh` before starting the stand. It preserves the +existing PostgreSQL and Valkey credentials and creates three independent S3 +identities plus the static server configuration. Credential values are never +printed. Repeated initialization keeps all existing keys. Set `S3_BUCKET` in +the environment or the local `.env` before the first initialization; changing +it later requires a planned migration of identities and retained data. + +The server loads its static identity secret before serving requests. There is +no anonymous identity. The bootstrap identity has administrative rights and +its individual key files are mounted only in `s3-init`. The API identity has +`Read:` and `Write:`; it cannot list objects or administer +buckets. The reconciler identity adds `List:`. All three identities +are necessarily present in the S3 server's own identity configuration secret. + +`s3-init` creates the bucket idempotently and waits for storage before API +replicas start. An anonymous 403 and bucket metadata alone do not prove that +retained volume locations have registered with the elected master. Bootstrap +first reads a persistent `.health/storage-v1` marker (creating it only when +absent), then verifies a fresh write/read/delete. This startup-only barrier +uses the caller's deadline, capped at 60 seconds; ordinary S3 operations and +API readiness do not retry through it. The marker is outside the reconciler's +`staging/` and `blobs/` namespaces. `scripts/restart.sh` already runs this +bootstrap before restoring API replicas. + +The bootstrap uses `/usr/local/bin/s3init`. The optional `recon` profile runs +`/usr/local/bin/reconcile`, with reconciler credentials mapped to the standard +`S3_ACCESS_KEY_ID_FILE` and `S3_SECRET_ACCESS_KEY_FILE` paths: + +```sh +docker compose --profile recon run --rm reconcile +docker compose --profile recon run --rm reconcile --apply +``` + +The API receives its own standard S3 key files. Its background reconciliation +worker must construct a separate store from +`S3_RECONCILE_ACCESS_KEY_ID_FILE` and `S3_RECONCILE_SECRET_ACCESS_KEY_FILE`; +listing with the API identity is denied. S3 is intentionally absent from API +readiness so text traffic can remain available during an object-store outage. + +The S3 cgroup is limited to one CPU, 512 MiB, and 256 processes. Upload limits +are four concurrent requests and 64 MiB in flight; reader buffers have a +32 MiB budget and the chunk cache is disabled. Disk allocation is capped at +64 volumes of approximately 128 MiB each (with normal volume rollover +overhead); this is a lab capacity limit, not a filesystem quota. + +Each API replica has 2 GiB and `GOMEMLIMIT=1536MiB`, with one image decoder. +Pixels alone are not the full budget: progressive JPEG retains coefficients, +and a wide RGBA64 PNG keeps two large scanlines beside its pixel buffer. +The 40 MP RGB/CMYK progressive JPEG and wide-PNG measurements are recorded in +`evidence/decoder-memory-*.json`; this does not change the S3 service's 512 MiB limit. + +Verify the adapter and real server with Docker running: + +```sh +go test -race -count=1 ./internal/platform/s3store +``` + +`testenv.StartS3(ctx)` returns `Store` with reconciler rights, `APIStore`, +`Admin`, endpoint and credentials. `Recreate(ctx)` replaces only its own +container while keeping its named volume and does not recreate the bucket. +It waits for the same retained-marker and write/read/delete barrier before +returning. `Start(ctx)` does so as well. Always call `Close()` to remove the +fixture's container and volume. diff --git a/infra/s3/seaweedfs.options b/infra/s3/seaweedfs.options new file mode 100644 index 00000000..db8c1c1f --- /dev/null +++ b/infra/s3/seaweedfs.options @@ -0,0 +1,31 @@ +dir=/data +ip=127.0.0.1 +ip.bind=127.0.0.1 +master.telemetry=false +master.volumeSizeLimitMB=128 +volume.port=9340 +volume.max=64 +volume.index=leveldb +volume.fileSizeLimitMB=10 +volume.concurrentUploadLimitMB=64 +volume.concurrentDownloadLimitMB=64 +volume.readBufferSizeMB=1 +volume.preStopSeconds=1 +filer=true +filer.exposeDirectoryData=false +filer.disableDirListing=true +filer.concurrentFileUploadLimit=4 +filer.concurrentUploadLimitMB=64 +s3=true +s3.ip.bind=0.0.0.0 +s3.port=8333 +s3.config=/run/secrets/s3_identities +s3.autoCreateBucket=false +s3.iam=false +s3.port.iceberg=0 +s3.port.lance=0 +s3.concurrentFileUploadLimit=4 +s3.concurrentUploadLimitMB=64 +s3.readerCacheSizeMB=32 +s3.cacheCapacityMB=0 +s3.idleTimeout=35 diff --git a/internal/app/app.go b/internal/app/app.go index 720a9ae5..3949add5 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -12,6 +12,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + "messenger/internal/attachment" + attachmentstore "messenger/internal/attachment/store" "messenger/internal/auth" authstore "messenger/internal/auth/store" "messenger/internal/chat" @@ -42,6 +44,9 @@ type Deps struct { InstanceID string // IdempotencyTTL zero means the contract's 24 hours. IdempotencyTTL time.Duration + Blobs attachment.Blobs + MaxUploadBytes int64 + MaxImagePixels int64 } // defaultRequestTimeout stays below the server's 10s write timeout, so a @@ -76,6 +81,9 @@ func New(d Deps) *App { auth.NewHandler(sessions).Register(public, protected) user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) + attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels) + binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions)) + attachment.NewHandler(attachments).Register(protected, binary) return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 02f8a33b..a9741ecc 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "net/url" "os" + "reflect" "sort" "strings" "sync" @@ -415,11 +416,11 @@ func TestMessagesEndToEnd(t *testing.T) { } page1 := ts.history(bob.token, room.ID, "?limit=1") - if len(page1.Items) != 1 || page1.NextCursor == nil || page1.Items[0] != first { + if len(page1.Items) != 1 || page1.NextCursor == nil || !reflect.DeepEqual(page1.Items[0], first) { t.Fatalf("page1 = %+v", page1) } page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor)) - if len(page2.Items) != 1 || page2.NextCursor != nil || page2.Items[0] != second { + if len(page2.Items) != 1 || page2.NextCursor != nil || !reflect.DeepEqual(page2.Items[0], second) { t.Fatalf("page2 = %+v", page2) } diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go new file mode 100644 index 00000000..2041d5e6 --- /dev/null +++ b/internal/app/attachment_test.go @@ -0,0 +1,171 @@ +package app_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "image" + "image/jpeg" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/app" + "messenger/internal/attachment" + "messenger/internal/chat" + "messenger/internal/testenv" +) + +func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized") +} + +func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) { + ts := newTester(t) + actor := ts.user("deadline_user", "User") + ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond}) + var held []*pgxpool.Conn + for range db.Pool.Config().MaxConns { + conn, err := db.Pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + held = append(held, conn) + } + defer func() { + for _, conn := range held { + conn.Release() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx) + req.Header.Set("Authorization", "Bearer "+actor.token) + rec := httptest.NewRecorder() + started := time.Now() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { + t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + } +} + +func (ts *tester) binary(method, path, token, mime string, data []byte, length int64) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(data)) + req.ContentLength = length + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if mime != "" { + req.Header.Set("Content-Type", mime) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + res := response{status: rec.Code, requestID: rec.Header().Get("X-Request-Id"), challenge: rec.Header().Get("WWW-Authenticate"), raw: rec.Body.Bytes()} + if res.requestID == "" || rec.Header().Get("X-Instance-Id") == "" { + ts.t.Fatal("binary response missing correlation headers") + } + return res +} + +func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) { + ctx := context.Background() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + ts := newTester(t) + ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, Blobs: s3.APIStore}) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "files", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/attachments?filename=" + url.QueryEscape("фото.png") + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))) + ts.expect(res, 201, "") + a := decode[attachment.Response](t, res) + digest := sha256.Sum256(data) + if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "ready" || a.ErrorCode != nil || len(a.Variants) != 0 { + t.Fatalf("attachment: %+v", a) + } + base := "/api/v1/attachments/" + a.ID + for _, suffix := range []string{"", "/content"} { + ts.expect(ts.do("GET", base+suffix, eve.token, nil, nil), 404, "not_found") + } + ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "") + got := ts.binary("GET", base+"/content", bob.token, "", nil, 0) + ts.expect(got, 200, "") + if !bytes.Equal(got.raw, data) { + t.Fatal("download bytes changed") + } + ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 400, "invalid_request") + msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()) + ts.expect(msg, 201, "") + if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID { + t.Fatal("lost attachment on message") + } + other := ts.chat(alice.token, "other", alice.ID) + ts.expect(ts.do("POST", "/api/v1/chats/"+other.ID+"/messages", alice.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()), 404, "not_found") + for _, tc := range []struct { + name, mime string + data []byte + length int64 + want int + }{ + {"MIME spoof", "image/jpeg", data, int64(len(data)), 415}, + {"broken body", "image/png", data[:len(data)/2], int64(len(data) / 2), 415}, + {"unsupported", "image/gif", data, int64(len(data)), 415}, + {"oversize", "image/png", nil, 10485761, 413}, + {"unknown length", "image/png", data, -1, 400}, + {"empty", "image/png", nil, 0, 400}, + } { + t.Run(tc.name, func(t *testing.T) { + r := ts.binary("POST", path, alice.token, tc.mime, tc.data, tc.length) + if r.status != tc.want { + t.Fatalf("got %d want %d: %s", r.status, tc.want, r.raw) + } + }) + } + for _, name := range []string{"../x.png", `a\b.png`, "bad\nname", strings.Repeat("я", 256)} { + ts.expect(ts.binary("POST", "/api/v1/chats/"+room.ID+"/attachments?filename="+url.QueryEscape(name), alice.token, "image/png", data, int64(len(data))), 400, "invalid_request") + } + var j bytes.Buffer + if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 201, "") + ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found") + // An interrupted read is never a 500 or a successfully published identity. + before := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&before); err != nil { + t.Fatal(err) + } + short := ts.binary("POST", path, alice.token, "image/png", data[:10], int64(len(data))) + if short.status != 400 && short.status != 503 { + t.Fatalf("interrupted body: %d %s", short.status, short.raw) + } + after := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after { + t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err) + } + if err := s3.Stop(ctx); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))), 503, "unavailable") + ts.expect(ts.do("GET", base+"/content", bob.token, nil, nil), 503, "unavailable") + ts.expect(ts.do("GET", "/health/ready", "", nil, nil), 200, "") + ts.message(alice.token, room.ID, fmt.Sprintf("text works with S3 down %d", time.Now().UnixNano())) +} diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go new file mode 100644 index 00000000..9a5cd171 --- /dev/null +++ b/internal/attachment/attachment.go @@ -0,0 +1,22 @@ +package attachment + +import ( + "context" + "github.com/google/uuid" + "time" +) + +type Attachment struct { + ID, ChatID, OwnerID uuid.UUID + Filename string + FileHash []byte + SizeBytes int64 + MIMEType, Status string + CreatedAt time.Time +} + +type Repository interface { + IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error) + Publish(context.Context, Attachment, string) error + GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error) +} diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go new file mode 100644 index 00000000..c211b36e --- /dev/null +++ b/internal/attachment/blob.go @@ -0,0 +1,25 @@ +package attachment + +import ( + "context" + "errors" + "io" + "time" +) + +var ErrObjectMissing = errors.New("object missing") + +type Object struct { + Key string + Size int64 + Modified time.Time +} + +type Blobs interface { + Put(context.Context, string, io.Reader, int64, string) error + Open(context.Context, string) (io.ReadCloser, error) + Stat(context.Context, string) (Object, error) + Copy(context.Context, string, string) error + Delete(context.Context, string) error + List(context.Context, string, string, int) ([]Object, error) +} diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go new file mode 100644 index 00000000..6420519f --- /dev/null +++ b/internal/attachment/handler.go @@ -0,0 +1,121 @@ +package attachment + +import ( + "encoding/hex" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" +) + +type Handler struct{ s *Service } + +func NewHandler(s *Service) *Handler { return &Handler{s} } +func (h *Handler) Register(protected, binary gin.IRouter) { + binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) + protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata)) + binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) +} + +type Response struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func toResponse(a Attachment) Response { + return Response{ + ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt), + } +} + +func (h *Handler) upload(c *gin.Context, actor uuid.UUID) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + httpx.Error(c, apperr.Invalid("chat_id must be a UUID")) + return + } + names := c.Request.URL.Query()["filename"] + if len(names) != 1 { + httpx.Error(c, apperr.Invalid("one filename is required")) + return + } + if err := ValidateFilename(names[0]); err != nil { + httpx.Error(c, err) + return + } + media, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || len(params) > 0 { + httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes) + a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toResponse(a)) +} +func attachmentID(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("attachment_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("attachment_id must be a UUID") + } + return id, nil +} +func (h *Handler) metadata(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + a, err := h.s.Get(c.Request.Context(), actor, id) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toResponse(a)) +} +func (h *Handler) content(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + variant, present := c.GetQuery("variant") + if present && variant != "original" { + httpx.Error(c, apperr.Invalid("variant must be original")) + return + } + a, r, err := h.s.Content(c.Request.Context(), actor, id) + if err != nil { + httpx.Error(c, err) + return + } + defer r.Close() + c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Type", a.MIMEType) + c.Header("Content-Length", strconv.FormatInt(a.SizeBytes, 10)) + c.Status(http.StatusOK) + if _, err := io.CopyN(c.Writer, r, a.SizeBytes); err != nil { + slog.Error("download interrupted", "request_id", httpx.RequestID(c), "error", err) + } +} diff --git a/internal/attachment/lock.go b/internal/attachment/lock.go new file mode 100644 index 00000000..31b73342 --- /dev/null +++ b/internal/attachment/lock.go @@ -0,0 +1,23 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + + "github.com/jackc/pgx/v5" +) + +// LockHash serializes CAS publication and collection until the transaction ends. +// A collision in the signed first eight digest bytes only adds serialization. +func LockHash(ctx context.Context, tx pgx.Tx, hash []byte) error { + if len(hash) != sha256.Size { + return errors.New("attachment: lock requires a SHA256 hash") + } + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(hash[:8]))) + return err +} + +func BlobKey(hash []byte) string { return "blobs/" + hex.EncodeToString(hash) } diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 00000000..729e42f6 --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,479 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + reconcilePageSize = 500 + reconcileOperationTimeout = 30 * time.Second +) + +type Finding struct { + Kind string `json:"kind"` + Key string `json:"key"` + Action string `json:"action"` + Applied bool `json:"applied"` +} + +// SQLQueries counts SQL statements, including transaction control and locks. +// S3Calls counts calls to the Blobs port, including each list page and deep GET. +type Stats struct { + ElapsedMS int64 `json:"elapsed_ms"` + SQLQueries int64 `json:"sql_queries"` + S3Calls int64 `json:"s3_calls"` + ScannedFiles int64 `json:"scanned_files"` + ScannedObjects int64 `json:"scanned_objects"` + Findings int64 `json:"findings"` + Repaired int64 `json:"repaired"` +} + +type Reconciler struct { + db *pgxpool.Pool + blobs Blobs + grace time.Duration +} + +func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) { + if db == nil || blobs == nil { + return nil, errors.New("reconcile: database and blobs are required") + } + if grace == 0 { + grace = time.Hour + } + if grace < 2*time.Minute { + return nil, errors.New("reconcile: grace must be at least two minutes") + } + return &Reconciler{db: db, blobs: blobs, grace: grace}, nil +} + +// Run streams findings and keeps at most one page of metadata in memory. +// Dry runs only read storage. Each apply candidate is checked again under the +// publisher's transaction lock; no scan snapshot authorizes a later mutation. +func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) { + started := time.Now() + defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }() + run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)} + for _, prefix := range []string{"staging/", "blobs/"} { + if err = run.objects(prefix); err != nil { + return stats, err + } + } + err = run.files() + return stats, err +} + +type reconcileRun struct { + r *Reconciler + ctx context.Context + apply, deep bool + emit func(Finding) + stats *Stats + cutoff time.Time +} + +func (s *reconcileRun) note(f *Finding) { + if f == nil { + return + } + s.stats.Findings++ + if f.Applied { + s.stats.Repaired++ + } + if s.emit != nil { + s.emit(*f) + } +} + +func (s *reconcileRun) old(modified time.Time) bool { + // Unknown timestamps must never authorize collection. + return !modified.IsZero() && modified.Before(s.cutoff) +} + +func (s *reconcileRun) objects(prefix string) error { + after := "" + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + page, err := s.r.blobs.List(opCtx, prefix, after, reconcilePageSize) + cancel() + if err != nil { + return fmt.Errorf("reconcile list %s: %w", prefix, err) + } + if len(page) == 0 { + return nil + } + var known map[string]bool + if prefix == "blobs/" { + known, err = s.knownFiles(page) + if err != nil { + return err + } + } + for _, obj := range page { + if obj.Key <= after { + return errors.New("reconcile: object page did not advance") + } + after = obj.Key + s.stats.ScannedObjects++ + if !s.old(obj.Modified) { + continue + } + var finding *Finding + if prefix == "staging/" { + finding, err = s.staging(obj) + } else { + finding, err = s.orphan(obj, known[obj.Key]) + } + if err != nil { + return fmt.Errorf("reconcile %s: %w", obj.Key, err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) stat(key string) (Object, error) { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + return s.r.blobs.Stat(opCtx, key) +} + +func (s *reconcileRun) delete(key string) error { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + err := s.r.blobs.Delete(opCtx, key) + if errors.Is(err, ErrObjectMissing) { + return nil + } + return err +} + +func (s *reconcileRun) staging(obj Object) (*Finding, error) { + f := &Finding{Kind: "stale_staging", Key: obj.Key, Action: "delete_object"} + if !s.apply { + return f, nil + } + current, err := s.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err = s.delete(obj.Key); err != nil { + return nil, err + } + f.Applied = true + return f, nil +} + +func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { + hashes := make([][]byte, 0, len(page)) + for _, obj := range page { + if !s.old(obj.Modified) { + continue + } + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err == nil && len(hash) == sha256.Size && BlobKey(hash) == obj.Key { + hashes = append(hashes, hash) + } + } + known := make(map[string]bool, len(hashes)) + if len(hashes) == 0 { + return known, nil + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var hash []byte + if err := rows.Scan(&hash); err != nil { + return nil, err + } + known[BlobKey(hash)] = true + } + return known, rows.Err() +} + +func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) { + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key { + return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil + } + if known { + return nil, nil + } + f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"} + if !s.apply { + return f, nil + } + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var exists bool + s.stats.SQLQueries++ + if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil { + return nil, err + } + if exists { + return nil, nil + } + current, err := op.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err = op.delete(obj.Key); err != nil { + return nil, err + } + f.Applied = true + return f, nil + }) +} + +// locked also covers hashes without a files row: a publisher and an orphan +// collector therefore cannot pass each other's existence check concurrently. +func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Finding, error)) (*Finding, error) { + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + s.stats.SQLQueries++ + tx, err := s.r.db.Begin(opCtx) + if err != nil { + return nil, err + } + committed := false + defer func() { + if !committed { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s.stats.SQLQueries++ + _ = tx.Rollback(cleanup) + } + }() + // Pool defaults cover ordinary requests; object I/O can outlive those + // idle limits. The operation deadline bounds the entire locked candidate. + s.stats.SQLQueries++ + if _, err = tx.Exec(opCtx, `SET LOCAL idle_in_transaction_session_timeout='35s'`); err != nil { + return nil, err + } + s.stats.SQLQueries++ + if err = LockHash(opCtx, tx, hash); err != nil { + return nil, err + } + f, err := fn(opCtx, tx) + if err != nil { + return nil, err + } + s.stats.SQLQueries++ + err = tx.Commit(opCtx) + committed = true // pgx closes the transaction even when COMMIT fails. + if err != nil { + return nil, err + } + return f, nil +} + +type reconcileFile struct { + hash []byte + size int64 + status string + created time.Time + referenced bool +} + +func (s *reconcileRun) files() error { + after := []byte{} + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, + EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) + FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after) + if err != nil { + return err + } + page := make([]reconcileFile, 0, reconcilePageSize) + for rows.Next() { + var f reconcileFile + if err = rows.Scan(&f.hash, &f.size, &f.status, &f.created, &f.referenced); err != nil { + rows.Close() + return err + } + page = append(page, f) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + // Close the page's SQL rows before acquiring a transaction connection. + for _, f := range page { + after = f.hash + s.stats.ScannedFiles++ + if !s.candidate(f) { + continue + } + var finding *Finding + if s.apply { + finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var current reconcileFile + s.stats.SQLQueries++ + err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, + EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) + FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash). + Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.candidate(current) { + return nil, nil + } + return op.inspect(current, tx) + }) + } else { + finding, err = s.inspect(f, nil) + } + if err != nil { + return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) candidate(f reconcileFile) bool { + if !s.old(f.created) { + return false + } + if !f.referenced { + return f.status == "pending" || f.status == "ready" + } + return f.status == "ready" || f.status == "failed" +} + +func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { + key := BlobKey(f.hash) + if !f.referenced { + // A publication that rolled back can leave fresh bytes behind an old + // pending row. Both the metadata and the current object must be old. + obj, err := s.stat(key) + if err != nil && !errors.Is(err, ErrObjectMissing) { + return nil, err + } + if err == nil && !s.old(obj.Modified) { + return nil, nil + } + finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"} + if !s.apply { + return finding, nil + } + // S3 must succeed first. A failed SQL commit then leaves a missing, + // unreferenced row that the next run can safely finish collecting. + if err := s.delete(key); err != nil { + return nil, err + } + s.stats.SQLQueries++ + if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + } + obj, err := s.stat(key) + if errors.Is(err, ErrObjectMissing) { + if f.status == "failed" { + return nil, nil + } + finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"} + if s.apply { + s.stats.SQLQueries++ + if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil { + return nil, err + } + finding.Applied = true + } + return finding, nil + } + if err != nil { + return nil, err + } + if obj.Size != f.size { + return &Finding{Kind: "size_mismatch", Key: key, Action: "report"}, nil + } + if s.deep { + match, err := s.verify(f) + if err != nil { + return nil, err + } + if !match { + return &Finding{Kind: "hash_mismatch", Key: key, Action: "report"}, nil + } + } + // A failed file stays failed even when the key reappears. Recovery requires + // an explicit publisher; a shallow HEAD cannot prove content integrity. + if f.status == "failed" { + return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil + } + return nil, nil +} + +func (s *reconcileRun) verify(f reconcileFile) (bool, error) { + if f.size < 0 || f.size == math.MaxInt64 { + return false, errors.New("reconcile: invalid file size") + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + body, err := s.r.blobs.Open(opCtx, BlobKey(f.hash)) + if err != nil { + return false, err + } + hash := sha256.New() + n, readErr := io.Copy(hash, io.LimitReader(body, f.size+1)) + closeErr := body.Close() + if readErr != nil { + return false, readErr + } + if closeErr != nil { + return false, closeErr + } + return n == f.size && bytes.Equal(hash.Sum(nil), f.hash), nil +} diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go new file mode 100644 index 00000000..39fd6dcd --- /dev/null +++ b/internal/attachment/reconcile_test.go @@ -0,0 +1,708 @@ +package attachment_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func TestReconcileBlobKey(t *testing.T) { + hash, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + if got := attachment.BlobKey(hash); got != "blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Fatalf("blob key: %q", got) + } +} + +// Only object storage is doubled: SQL, transactions, FKs and advisory locks are real. +type reconcileBlobs struct { + mu sync.Mutex + objects map[string]attachment.Object + bodies map[string][]byte + statErr, deleteErr, openErr error + listed func(string) + stating func(string) + opening func(string) io.ReadCloser + writes, readBytes int + limits []int +} + +func newReconcileBlobs() *reconcileBlobs { + return &reconcileBlobs{objects: make(map[string]attachment.Object), bodies: make(map[string][]byte)} +} +func (b *reconcileBlobs) add(key string, body []byte, modified time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + b.objects[key] = attachment.Object{Key: key, Size: int64(len(body)), Modified: modified} + b.bodies[key] = bytes.Clone(body) +} +func (b *reconcileBlobs) has(key string) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.objects[key] + return ok +} +func (b *reconcileBlobs) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error { + body, err := io.ReadAll(io.LimitReader(r, size+1)) + if err != nil { + return err + } + if int64(len(body)) != size { + return io.ErrUnexpectedEOF + } + b.add(key, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Copy(_ context.Context, source, dest string) error { + b.mu.Lock() + body, ok := b.bodies[source] + b.mu.Unlock() + if !ok { + return attachment.ErrObjectMissing + } + b.add(dest, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Open(_ context.Context, key string) (io.ReadCloser, error) { + if b.openErr != nil { + return nil, b.openErr + } + if b.opening != nil { + return b.opening(key), nil + } + b.mu.Lock() + defer b.mu.Unlock() + body, ok := b.bodies[key] + if !ok { + return nil, attachment.ErrObjectMissing + } + return io.NopCloser(bytes.NewReader(body)), nil +} +func (b *reconcileBlobs) Stat(_ context.Context, key string) (attachment.Object, error) { + if b.stating != nil { + b.stating(key) + } + b.mu.Lock() + defer b.mu.Unlock() + if b.statErr != nil { + return attachment.Object{}, b.statErr + } + obj, ok := b.objects[key] + if !ok { + return attachment.Object{}, fmt.Errorf("head: %w", attachment.ErrObjectMissing) + } + return obj, nil +} +func (b *reconcileBlobs) Delete(_ context.Context, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.deleteErr != nil { + return b.deleteErr + } + delete(b.objects, key) + delete(b.bodies, key) + b.writes++ + return nil +} +func (b *reconcileBlobs) List(_ context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + b.mu.Lock() + b.limits = append(b.limits, limit) + if limit < 1 || limit > 500 { + b.mu.Unlock() + return nil, errors.New("unbounded page") + } + var page []attachment.Object + for key, obj := range b.objects { + if strings.HasPrefix(key, prefix) && key > after { + page = append(page, obj) + } + } + sort.Slice(page, func(i, j int) bool { return page[i].Key < page[j].Key }) + if len(page) > limit { + page = page[:limit] + } + b.mu.Unlock() + if b.listed != nil { + b.listed(prefix) + } + return page, nil +} + +type reconcileFixture struct { + t *testing.T + db *pgxpool.Pool + b *reconcileBlobs + user, chat, message uuid.UUID +} + +func reconcileSetup(t *testing.T, db *testenv.DB) *reconcileFixture { + t.Helper() + db.Reset(t) + f := &reconcileFixture{t: t, db: db.Pool, b: newReconcileBlobs(), user: uuid.New(), chat: uuid.New(), message: uuid.New()} + f.exec(`INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,'reconcile','Reconcile','unused',now())`, f.user) + f.exec(`INSERT INTO chats(id,title,created_at) VALUES($1,'reconcile',now())`, f.chat) + f.exec(`INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, f.message, f.user) + return f +} +func (f *reconcileFixture) exec(sql string, args ...any) { + f.t.Helper() + if _, err := f.db.Exec(context.Background(), sql, args...); err != nil { + f.t.Fatal(err) + } +} +func (f *reconcileFixture) file(body, status string, age time.Duration, referenced, present bool) []byte { + f.t.Helper() + hash := sha256.Sum256([]byte(body)) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, time.Now().Add(-age)) + if referenced { + id := uuid.New() + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'image.png',$4,now())`, id, f.chat, f.user, hash[:]) + f.exec(`INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) + VALUES($1,$2,$3,(SELECT count(*) FROM message_attachments WHERE chat_id=$1 AND message_id=$2))`, f.chat, f.message, id) + } + if present { + f.b.add(attachment.BlobKey(hash[:]), []byte(body), time.Now().Add(-age)) + } + return hash[:] +} +func (f *reconcileFixture) status(hash []byte) string { + f.t.Helper() + var s string + err := f.db.QueryRow(context.Background(), `SELECT status FROM files WHERE hash=$1`, hash).Scan(&s) + if errors.Is(err, pgx.ErrNoRows) { + return "" + } + if err != nil { + f.t.Fatal(err) + } + return s +} +func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment.Finding, error) { + f.t.Helper() + r, err := attachment.NewReconciler(f.db, f.b, 0) + if err != nil { + f.t.Fatal(err) + } + var found []attachment.Finding + stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) }) + return stats, found, err +} + +func TestReconciler(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("constructor validates and defaults grace", func(t *testing.T) { + f := reconcileSetup(t, db) + for _, grace := range []time.Duration{-time.Second, time.Minute} { + if _, err := attachment.NewReconciler(f.db, f.b, grace); err == nil { + t.Fatalf("accepted grace %v", grace) + } + } + if _, err := attachment.NewReconciler(nil, f.b, 0); err == nil { + t.Fatal("accepted nil DB") + } + if _, err := attachment.NewReconciler(f.db, nil, 0); err == nil { + t.Fatal("accepted nil blobs") + } + if _, err := attachment.NewReconciler(f.db, f.b, 2*time.Minute); err != nil { + t.Fatal(err) + } + fresh := f.file("half hour", "ready", 30*time.Minute, false, true) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(fresh) != "ready" { + t.Fatalf("default grace: %+v %v", stats, err) + } + }) + t.Run("dry run then apply then repeat preserves fresh referenced and message links", func(t *testing.T) { + f := reconcileSetup(t, db) + old := 2 * time.Hour + pending := f.file("pending", "pending", old, false, true) + unused := f.file("unused", "ready", old, false, true) + used := f.file("used", "ready", old, true, true) + fresh := f.file("fresh", "pending", time.Minute, false, true) + missing := f.file("missing", "ready", old, true, false) + f.b.add("staging/old", []byte("old"), time.Now().Add(-old)) + f.b.add("staging/fresh", []byte("new"), time.Now()) + orphan := sha256.Sum256([]byte("orphan")) + orphanKey := attachment.BlobKey(orphan[:]) + f.b.add(orphanKey, []byte("orphan"), time.Now().Add(-old)) + stats, findings, err := f.run(false, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 5 || len(findings) != 5 { + t.Fatalf("dry: %+v %v %+v", stats, err, findings) + } + if f.b.writes != 0 || f.status(pending) != "pending" || f.status(missing) != "ready" { + t.Fatal("dry run mutated storage") + } + for _, finding := range findings { + if finding.Applied { + t.Fatal("dry finding marked applied") + } + } + stats, findings, err = f.run(true, false) + if err != nil || stats.Repaired != 5 { + t.Fatalf("apply: %+v %v %+v", stats, err, findings) + } + if f.status(pending) != "" || f.status(unused) != "" || f.status(missing) != "failed" { + t.Fatal("metadata repair incorrect") + } + if f.b.has(orphanKey) || f.b.has("staging/old") || !f.b.has("staging/fresh") || !f.b.has(attachment.BlobKey(used)) || !f.b.has(attachment.BlobKey(fresh)) { + t.Fatal("incorrect object collection") + } + var links int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM message_attachments`).Scan(&links); err != nil || links != 2 { + t.Fatalf("message links lost: %d %v", links, err) + } + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 { + t.Fatalf("repeat: %+v %v", stats, err) + } + }) + t.Run("outage is not missing and failed delete keeps row", func(t *testing.T) { + f := reconcileSetup(t, db) + used := f.file("used", "ready", 2*time.Hour, true, true) + outage := errors.New("injected S3 outage") + f.b.statErr = outage + stats, _, err := f.run(true, false) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(used) != "ready" { + t.Fatalf("outage: %+v %v", stats, err) + } + f.b.statErr = nil + unused := f.file("unused", "pending", 2*time.Hour, false, true) + f.b.deleteErr = outage + _, _, err = f.run(true, false) + if !errors.Is(err, outage) || f.status(unused) != "pending" { + t.Fatalf("delete failure: %v", err) + } + }) + t.Run("deep reports corruption and never revives failed without verification", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("evil"), time.Now().Add(-2*time.Hour)) + stats, found, err := f.run(true, true) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("deep: %+v %v %+v", stats, err, found) + } + f.exec(`UPDATE files SET status='failed' WHERE hash=$1`, hash) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("shallow revived failed file") + } + stats, _, err = f.run(true, true) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("corrupt file revived") + } + }) + t.Run("wrong size reports only and fresh missing stays ready", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("sized", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("longer body"), time.Now().Add(-2*time.Hour)) + fresh := f.file("new missing", "ready", time.Minute, true, false) + stats, found, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 1 || found[0].Kind != "size_mismatch" || f.status(fresh) != "ready" { + t.Fatalf("size/fresh: %+v %v %+v", stats, err, found) + } + }) + t.Run("orphan rechecks current object age", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("orphan")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("orphan"), time.Now().Add(-2*time.Hour)) + f.b.listed = func(prefix string) { + if prefix == "blobs/" { + f.b.add(key, []byte("republished"), time.Now()) + } + } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || !f.b.has(key) { + t.Fatalf("fresh replacement deleted: %+v %v", stats, err) + } + }) + t.Run("row rechecks references after taking lock", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("race refs", "ready", 2*time.Hour, false, true) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, true, false, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'race.png',$4,now())`, uuid.New(), f.chat, f.user, hash) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatal("new reference collected") + } + }) + t.Run("concurrent upload lock prevents orphan collection", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("publishing")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("publishing"), time.Now().Add(-2*time.Hour)) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash[:]); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, true, false, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + if _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,10,'image/png','ready')`, hash[:]); err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if !f.b.has(key) || f.status(hash[:]) != "ready" { + t.Fatal("published blob collected") + } + }) + t.Run("signed big endian lock key matches publisher", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := make([]byte, 32) + for i := 0; i < 8; i++ { + hash[i] = 255 + } + hash[7] = 254 + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + other, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer other.Rollback(ctx) + var acquired bool + if err = other.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(-2::bigint)`).Scan(&acquired); err != nil || acquired { + t.Fatalf("lock mismatch: %v %v", acquired, err) + } + }) + t.Run("keyset pages tolerate deletes and report costs", func(t *testing.T) { + f := reconcileSetup(t, db) + for i := 0; i < 1003; i++ { + f.b.add(fmt.Sprintf("staging/%04d", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','pending',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + stats, _, err := f.run(true, false) + if err != nil || stats.ScannedFiles != 1003 || stats.ScannedObjects != 1003 || stats.Repaired != 2006 { + t.Fatalf("pages: %+v %v", stats, err) + } + var n int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&n); err != nil || n != 0 { + t.Fatalf("skipped rows: %d %v", n, err) + } + if len(f.b.limits) < 4 || stats.SQLQueries == 0 || stats.S3Calls == 0 { + t.Fatalf("missing paging/costs: %+v", stats) + } + raw, err := json.Marshal(stats) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"elapsed_ms", "sql_queries", "s3_calls", "scanned_files", "scanned_objects", "findings", "repaired"} { + if !bytes.Contains(raw, []byte(`"`+key+`":`)) { + t.Fatalf("missing JSON key %s: %s", key, raw) + } + } + }) + t.Run("old blob existence checks are batched by page", func(t *testing.T) { + f := reconcileSetup(t, db) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','ready',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),$1,$2,'batch.png',hash,now() FROM files`, f.chat, f.user) + for i := 1; i <= 1003; i++ { + f.b.add(fmt.Sprintf("blobs/%064x", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + stats, _, err := f.run(false, false) + if err != nil || stats.SQLQueries > 8 || stats.SQLQueries == 0 || stats.ScannedFiles != 1003 || stats.Findings != 0 { + t.Fatalf("per-object SQL lookup: %+v %v", stats, err) + } + }) +} + +func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) { + t.Helper() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + for { + var waiting bool + if err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted)`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + return + } + select { + case <-deadline.C: + t.Fatal("reconciler did not wait on publisher advisory lock") + case <-tick.C: + } + } +} + +type reconcileEndless struct { + read int64 + closed bool +} + +func (r *reconcileEndless) Read(p []byte) (int, error) { + for i := range p { + p[i] = 'x' + } + r.read += int64(len(p)) + return len(p), nil +} +func (r *reconcileEndless) Close() error { r.closed = true; return nil } + +func TestReconcilerFailureBoundaries(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("slow object call survives pool idle transaction timeout", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("slow object", "pending", 2*time.Hour, false, true) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "100ms" + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + f.db = pool + f.b.stating = func(string) { time.Sleep(250 * time.Millisecond) } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("slow S3 transaction rolled back: %+v %v", stats, err) + } + }) + t.Run("fresh object with old unused metadata is preserved", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("republished", "pending", 2*time.Hour, false, true) + f.b.add(attachment.BlobKey(hash), []byte("republished"), time.Now()) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "pending" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("fresh blob lost: %+v %v", stats, err) + } + }) + t.Run("deep read stops at size plus one and closes body", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + body := &reconcileEndless{} + f.b.opening = func(string) io.ReadCloser { return body } + stats, found, err := f.run(true, true) + if err != nil || body.read != 5 || !body.closed || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" { + t.Fatalf("unbounded deep read: read=%d closed=%v %+v %v", body.read, body.closed, stats, err) + } + }) + t.Run("deep GET outage preserves referenced state", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + outage := errors.New("injected GET outage") + f.b.openErr = outage + stats, _, err := f.run(true, true) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("GET outage: %+v %v", stats, err) + } + }) + t.Run("valid failed object remains stable even deep", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "failed", 2*time.Hour, true, true) + for _, deep := range []bool{false, true} { + stats, found, err := f.run(true, deep) + if err != nil || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "failed_present" || f.status(hash) != "failed" { + t.Fatalf("failed state unstable: %+v %v %+v", stats, err, found) + } + } + }) + t.Run("dry run works with a read only PostgreSQL session", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("missing", "ready", 2*time.Hour, true, false) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + readOnly, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer readOnly.Close() + r, err := attachment.NewReconciler(readOnly, f.b, 0) + if err != nil { + t.Fatal(err) + } + stats, err := r.Run(ctx, false, true, nil) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("read only dry run: %+v %v", stats, err) + } + }) + t.Run("failed SQL commit after blob deletion is retryable", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("unused", "pending", 2*time.Hour, false, true) + f.exec(`CREATE FUNCTION reconcile_reject_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected commit failure'; END $$`) + f.exec(`CREATE CONSTRAINT TRIGGER reconcile_reject_delete AFTER DELETE ON files DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reconcile_reject_delete()`) + t.Cleanup(func() { + f.exec(`DROP TRIGGER IF EXISTS reconcile_reject_delete ON files`) + f.exec(`DROP FUNCTION IF EXISTS reconcile_reject_delete()`) + }) + stats, _, err := f.run(true, false) + if err == nil || stats.Repaired != 0 || f.status(hash) != "pending" || f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("failed commit: %+v %v", stats, err) + } + f.exec(`DROP TRIGGER reconcile_reject_delete ON files`) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("retry: %+v %v", stats, err) + } + }) +} + +func TestReconcilerS3(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, false) + missing := f.file("missing", "ready", 2*time.Hour, true, false) + key := attachment.BlobKey(hash) + if err = s3.Store.Put(ctx, key, strings.NewReader("evil"), 4, "image/png"); err != nil { + t.Fatal(err) + } + if err = s3.Store.Put(ctx, "staging/fresh", strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, s3.Store, 0) + if err != nil { + t.Fatal(err) + } + // Exercise the command with file-based configuration and the real listing + // identity. Its default must remain read-only, including missing metadata. + secretDir := t.TempDir() + cliEnv := os.Environ() + for name, value := range map[string]string{ + "DATABASE_URL_FILE": db.DSN, + "S3_ACCESS_KEY_ID_FILE": s3.AccessKey, + "S3_SECRET_ACCESS_KEY_FILE": s3.SecretKey, + } { + path := filepath.Join(secretDir, name) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + cliEnv = append(cliEnv, name+"="+path) + } + cliEnv = append(cliEnv, "S3_ENDPOINT="+s3.Endpoint, "S3_REGION="+s3.Region, "S3_BUCKET="+s3.Bucket) + cmd := exec.CommandContext(ctx, "go", "run", "../../cmd/reconcile", "--deep") + cmd.Env = cliEnv + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("CLI dry run: %v: %s", err, stderr.String()) + } + decoder := json.NewDecoder(&stdout) + var cliStats attachment.Stats + for i := 0; i < 2; i++ { + var finding attachment.Finding + if err := decoder.Decode(&finding); err != nil || finding.Applied || finding.Kind == "" || finding.Key == "" || finding.Action == "" { + t.Fatalf("CLI finding: %+v %v", finding, err) + } + } + if err := decoder.Decode(&cliStats); err != nil || cliStats.Findings != 2 || cliStats.Repaired != 0 || f.status(missing) != "ready" { + t.Fatalf("CLI mutated by default: %+v %v", cliStats, err) + } + var found []attachment.Finding + stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) }) + if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" { + t.Fatalf("S3 dry: %+v %v %+v", stats, err, found) + } + stats, err = r.Run(ctx, true, true, nil) + if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" { + t.Fatalf("S3 apply: %+v %v", stats, err) + } + if _, err = s3.Store.Stat(ctx, key); err != nil { + t.Fatal("referenced corrupt object deleted", err) + } + if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil { + t.Fatal("fresh staging deleted", err) + } + stats, err = r.Run(ctx, true, true, nil) + if err != nil || stats.Repaired != 0 { + t.Fatalf("S3 repeat: %+v %v", stats, err) + } +} + +func TestReconcileLockRejectsMalformedHash(t *testing.T) { + if err := attachment.LockHash(context.Background(), nil, []byte("bad")); err == nil { + t.Fatal("malformed hash accepted") + } +} diff --git a/internal/attachment/schema_test.go b/internal/attachment/schema_test.go new file mode 100644 index 00000000..c5db7ee6 --- /dev/null +++ b/internal/attachment/schema_test.go @@ -0,0 +1,27 @@ +package attachment_test + +import ( + "context" + "messenger/internal/testenv" + "testing" +) + +func TestFilesSchemaRejectsMalformedHashes(t *testing.T) { + db, err := testenv.StartPostgres(context.Background()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err != nil { + t.Fatalf("valid content-addressed file: %v", err) + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ('bad',12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted a malformed SHA256") + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted duplicate content hash") + } +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 00000000..ee574176 --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,172 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "errors" + "io" + "log/slog" + "net/http" + "time" + + "github.com/google/uuid" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +type Service struct { + repo Repository + blobs Blobs + maxBytes, maxPixels int64 + decode chan struct{} +} + +func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service { + if maxBytes == 0 { + maxBytes = MaxUploadBytes + } + if maxPixels == 0 { + maxPixels = MaxImagePixels + } + return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)} +} + +func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) { + if err := ValidateFilename(filename); err != nil { + return Attachment{}, err + } + if mime != "image/png" && mime != "image/jpeg" { + return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted") + } + ok, err := s.repo.IsMember(ctx, chatID, actor) + if err != nil { + return Attachment{}, err + } + if !ok { + return Attachment{}, apperr.NotFound("chat not found") + } + if size > s.maxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if size <= 0 { + return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required") + } + if s.blobs == nil { + return Attachment{}, apperr.Unavailable("object storage is unavailable") + } + id := ident.New() + stage := "staging/" + id.String() + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := s.blobs.Delete(cleanup, stage); err != nil { + slog.Warn("staging cleanup deferred", "error", err) + } + }() + hash := sha256.New() + counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)} + err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime) + if err != nil { + _, tooLarge := errors.AsType[*http.MaxBytesError](err) + if counter.n > s.maxBytes || tooLarge { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + return Attachment{}, apperr.Invalid("incomplete upload") + } + return Attachment{}, unavailable(err) + } + var extra [1]byte + n, readErr := counter.Read(extra[:]) + if counter.n > s.maxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) { + return Attachment{}, apperr.Invalid("body length does not match Content-Length") + } + // Full decoding is bounded by dimensions and one decoder per replica. Body + // transfer itself stays streaming and does not occupy the decoder slot. + select { + case s.decode <- struct{}{}: + case <-ctx.Done(): + return Attachment{}, unavailable(ctx.Err()) + } + err = s.validate(ctx, stage, mime) + <-s.decode + if err != nil { + return Attachment{}, err + } + a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)} + if err := s.repo.Publish(ctx, a, stage); err != nil { + return Attachment{}, err + } + return a, nil +} + +func (s *Service) validate(ctx context.Context, stage, mime string) error { + r, err := s.blobs.Open(ctx, stage) + if err != nil { + return unavailable(err) + } + defer r.Close() + tracked := &storageReader{reader: r} + err = ValidateImage(tracked, mime, s.maxPixels) + if tracked.err != nil { + return unavailable(tracked.err) + } + if ctx.Err() != nil { + return unavailable(ctx.Err()) + } + return err +} + +// Image sniffing can replace a reader failure with ErrFormat. Preserve the +// storage failure independently, while ordinary EOF still means malformed bytes. +type storageReader struct { + reader io.Reader + err error +} + +func (r *storageReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil && !errors.Is(err, io.EOF) { + r.err = err + } + return n, err +} + +func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) { + a, err := s.repo.GetForMember(ctx, id, actor) + if err != nil { + return Attachment{}, err + } + if a.Status != "ready" { + return Attachment{}, apperr.Conflict("attachment is not ready") + } + return a, nil +} + +func (s *Service) Content(ctx context.Context, actor, id uuid.UUID) (Attachment, io.ReadCloser, error) { + a, err := s.Get(ctx, actor, id) + if err != nil { + return Attachment{}, nil, err + } + if s.blobs == nil { + return Attachment{}, nil, apperr.Unavailable("object storage is unavailable") + } + r, err := s.blobs.Open(ctx, BlobKey(a.FileHash)) + if err != nil { + return Attachment{}, nil, unavailable(err) + } + return a, r, nil +} + +type countReader struct { + r io.Reader + n int64 +} + +func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e } +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) +} diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go new file mode 100644 index 00000000..c207a956 --- /dev/null +++ b/internal/attachment/service_test.go @@ -0,0 +1,270 @@ +package attachment_test + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "messenger/internal/attachment" + "messenger/internal/attachment/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" +) + +type testBlobs struct { + mu sync.Mutex + data map[string][]byte + copies int + failCopy bool +} + +type blockingPublication struct { + *testBlobs + entered, resume chan struct{} +} + +func (b *blockingPublication) Copy(ctx context.Context, from, to string) error { + close(b.entered) + select { + case <-b.resume: + case <-ctx.Done(): + return ctx.Err() + } + return b.testBlobs.Copy(ctx, from, to) +} +func (b *blockingPublication) Stat(ctx context.Context, key string) (attachment.Object, error) { + o, e := b.testBlobs.Stat(ctx, key) + o.Modified = time.Now().Add(-2 * time.Hour) + return o, e +} + +func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + actor, room := ident.New(), ident.New() + if _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'race_user','User','hash',now())`, actor); err != nil { + t.Fatal(err) + } + if _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'race',now())`, room); err != nil { + t.Fatal(err) + } + if _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, room, actor); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(data) + if _, err = db.Pool.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png','pending',now()-interval '2 hours')`, hash[:], len(data)); err != nil { + t.Fatal(err) + } + b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})} + s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0) + uploaded := make(chan error, 1) + go func() { + _, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data)) + uploaded <- e + }() + select { + case <-b.entered: + case <-ctx.Done(): + t.Fatal("publisher never reached copy") + } + r, err := attachment.NewReconciler(db.Pool, b, time.Hour) + if err != nil { + t.Fatal(err) + } + collected := make(chan error, 1) + go func() { + stats, e := r.Run(ctx, true, false, nil) + if e == nil && stats.Repaired != 0 { + e = errors.New("collector removed a newly referenced file") + } + collected <- e + }() + // Observe a real waiting advisory lock, rather than guessing which goroutine won. + deadline := time.Now().Add(3 * time.Second) + waiting := false + for time.Now().Before(deadline) { + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted`).Scan(&n); err != nil { + t.Fatal(err) + } + if n > 0 { + waiting = true + break + } + time.Sleep(5 * time.Millisecond) + } + close(b.resume) + if !waiting { + t.Fatal("collector did not serialize with the publisher") + } + if err := <-uploaded; err != nil { + t.Fatal(err) + } + if err := <-collected; err != nil { + t.Fatal(err) + } + if _, err := b.Stat(ctx, attachment.BlobKey(hash[:])); err != nil { + t.Fatal("collector deleted published data", err) + } +} + +func (b *testBlobs) Put(_ context.Context, k string, r io.Reader, n int64, m string) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + b.mu.Lock() + defer b.mu.Unlock() + b.data[k] = data + return nil +} +func (b *testBlobs) Open(_ context.Context, k string) (io.ReadCloser, error) { + b.mu.Lock() + defer b.mu.Unlock() + data, ok := b.data[k] + if !ok { + return nil, attachment.ErrObjectMissing + } + return io.NopCloser(bytes.NewReader(data)), nil +} +func (b *testBlobs) Stat(_ context.Context, k string) (attachment.Object, error) { + b.mu.Lock() + defer b.mu.Unlock() + data, ok := b.data[k] + if !ok { + return attachment.Object{}, attachment.ErrObjectMissing + } + return attachment.Object{Key: k, Size: int64(len(data)), Modified: time.Now()}, nil +} +func (b *testBlobs) Copy(_ context.Context, from, to string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.failCopy { + return errors.New("copy outage") + } + b.copies++ + b.data[to] = bytes.Clone(b.data[from]) + return nil +} +func (b *testBlobs) Delete(_ context.Context, k string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.data, k) + return nil +} +func (b *testBlobs) List(context.Context, string, string, int) ([]attachment.Object, error) { + return nil, nil +} + +func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { + ctx := context.Background() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + actor, c1, c2 := ident.New(), ident.New(), ident.New() + _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'alice','Alice','hash',now());`, actor) + if err != nil { + t.Fatal(err) + } + for _, c := range []uuid.UUID{c1, c2} { + _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'room',now())`, c) + if err != nil { + t.Fatal(err) + } + _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, c, actor) + if err != nil { + t.Fatal(err) + } + } + b := &testBlobs{data: map[string][]byte{}} + s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels) + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + var results [10]attachment.Attachment + var wg sync.WaitGroup + for i := range results { + wg.Go(func() { + c := c1 + if i%2 == 1 { + c = c2 + } + var e error + results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data)) + if e != nil { + t.Error(e) + } + }) + } + wg.Wait() + var files, atts int + err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments)`).Scan(&files, &atts) + if err != nil { + t.Fatal(err) + } + if files != 1 || atts != 10 || b.copies != 1 || len(b.data) != 1 { + t.Fatalf("files=%d attachments=%d copies=%d objects=%d", files, atts, b.copies, len(b.data)) + } + if results[0].ID == results[1].ID { + t.Fatal("deduplication merged chat ACL identity") + } + if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("outsider got %v", err) + } + // A removed blob is republished from the new upload, preserving old references. + b.mu.Lock() + clear(b.data) + b.mu.Unlock() + if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil { + t.Fatal(err) + } + if b.copies != 2 { + t.Fatalf("missing ready blob was not restored: copies=%d", b.copies) + } + b.failCopy = true + other := bytes.Clone(data) + other = append(other, 0) + if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("copy outage: %v", err) + } + if len(b.data) != 1 { + t.Fatalf("failed publication left staging: %d objects", len(b.data)) + } + b.failCopy = false + // S3 publication succeeds, then PostgreSQL refuses the logical attachment. + // Compensation may remove staging but must leave shared CAS data intact. + _, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$; + CREATE TRIGGER reject_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION reject_attachment()`) + if err != nil { + t.Fatal(err) + } + if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil { + t.Fatal("publication survived rejected metadata") + } + if len(b.data) != 2 { + t.Fatalf("expected original and recoverable orphan, got %d objects", len(b.data)) + } + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&files); err != nil || files != 1 { + t.Fatalf("failed DB insert committed file metadata: %d %v", files, err) + } +} diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go new file mode 100644 index 00000000..607932a9 --- /dev/null +++ b/internal/attachment/store/postgres.go @@ -0,0 +1,79 @@ +package store + +import ( + "context" + "errors" + "fmt" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" +) + +type Attachments struct { + pool *pgxpool.Pool + blobs attachment.Blobs +} + +func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments { + return &Attachments{pool, blobs} +} + +func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) { + var ok bool + err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok) + return ok, postgres.Classify(err) +} + +// Publication and garbage collection use the same hash lock. The transaction +// spans only publication, never client transfer or image decoding. +func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil { + return err + } + if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil { + return err + } + var status string + err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + copyNeeded := status != "ready" + if !copyNeeded { + obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash)) + if errors.Is(e, attachment.ErrObjectMissing) { + copyNeeded = true + } else if e != nil { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e) + } else if obj.Size != a.SizeBytes { + copyNeeded = true + } + } + if copyNeeded { + if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) + } + } + _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType) + if err != nil { + return fmt.Errorf("publish file: %w", err) + } + _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) + return err + })) +} + +func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) { + var a attachment.Attachment + err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at + FROM attachments a JOIN files f ON f.hash=a.file_hash + WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return attachment.Attachment{}, apperr.NotFound("attachment not found") + } + return a, postgres.Classify(err) +} diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go new file mode 100644 index 00000000..a810545f --- /dev/null +++ b/internal/attachment/validation.go @@ -0,0 +1,51 @@ +package attachment + +import ( + "bytes" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "strings" + "unicode" + "unicode/utf8" + + "messenger/internal/platform/apperr" +) + +const MaxUploadBytes int64 = 10 * 1024 * 1024 +const MaxImagePixels int64 = 40_000_000 + +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return apperr.Invalid("filename must contain 1-255 characters") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return apperr.Invalid("filename contains a forbidden character") + } + } + return nil +} + +// The small header is replayed into the decoder; pixels are allocated only +// after checking dimensions. The caller serializes decoding per replica. +func ValidateImage(r io.Reader, mime string, maxPixels int64) error { + limited := io.LimitReader(r, MaxUploadBytes+1) + var header bytes.Buffer + cfg, format, err := image.DecodeConfig(io.TeeReader(limited, &header)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + if (format != "png" && format != "jpeg") || "image/"+format != mime { + return apperr.UnsupportedMedia("image does not match Content-Type") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) { + return apperr.PayloadTooLarge("image exceeds pixel limit") + } + _, _, err = image.Decode(io.MultiReader(bytes.NewReader(header.Bytes()), limited)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + return nil +} diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go new file mode 100644 index 00000000..7ef3c509 --- /dev/null +++ b/internal/attachment/validation_test.go @@ -0,0 +1,93 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "messenger/internal/platform/apperr" + "strings" + "testing" +) + +func TestValidateFilename(t *testing.T) { + for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} { + if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%q: %v", name, err) + } + } + if err := ValidateFilename("фото 👋.png"); err != nil { + t.Fatal(err) + } +} + +type failingRead struct{ prefix []byte } + +func (r *failingRead) Read(p []byte) (int, error) { + if len(r.prefix) > 0 { + n := copy(p, r.prefix) + r.prefix = r.prefix[n:] + return n, nil + } + return 0, errors.New("injected storage read outage") +} + +type openedBlob struct { + Blobs + reader io.Reader +} + +func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(b.reader), nil +} + +func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) { + var pngData bytes.Buffer + if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil { + t.Fatal(err) + } + for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} { + s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0) + err := s.validate(context.Background(), "staging/test", "image/png") + if apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err) + } + } +} + +func TestValidateImageRejectsCorruptionAndLimitsPixelsBeforeDecode(t *testing.T) { + im := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + im.Set(1, 1, color.NRGBA{R: 200, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, im); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, mime string + data []byte + pixels int64 + kind apperr.Kind + }{ + {"valid", "image/png", b.Bytes(), 6, apperr.KindInternal}, + {"spoof", "image/jpeg", b.Bytes(), 6, apperr.KindUnsupportedMedia}, + {"truncated", "image/png", b.Bytes()[:len(b.Bytes())-20], 6, apperr.KindUnsupportedMedia}, + {"pixels", "image/png", b.Bytes(), 5, apperr.KindPayloadTooLarge}, + {"garbage", "image/png", []byte("not a picture"), 6, apperr.KindUnsupportedMedia}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateImage(bytes.NewReader(tc.data), tc.mime, tc.pixels) + if tc.name == "valid" { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil || apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want kind %v", err, tc.kind) + } + }) + } +} diff --git a/internal/chat/attachment_handler_test.go b/internal/chat/attachment_handler_test.go new file mode 100644 index 00000000..764432f5 --- /dev/null +++ b/internal/chat/attachment_handler_test.go @@ -0,0 +1,72 @@ +package chat_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/httpx" +) + +func messageRouter(f fixture, actor uuid.UUID) *gin.Engine { + r := gin.New() + r.Use(func(c *gin.Context) { httpx.SetUserID(c, actor) }) + chat.NewHandler(f.svc).Register(r) + return r +} + +func sendJSON(r http.Handler, chatID uuid.UUID, key, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/chats/"+chatID.String()+"/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", key) + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + return rec +} + +func TestMessageAttachmentJSONEmptyArrayAndOmitted(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for i, body := range []string{`{"text":"hello","attachment_ids":[]}`, `{"text":"hello"}`} { + rec := sendJSON(r, c, fmt.Sprint(i), body) + if rec.Code != http.StatusCreated { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &fields); err != nil { + t.Fatal(err) + } + if string(fields["attachment_ids"]) != "[]" { + t.Fatalf("attachment_ids = %s, want []", fields["attachment_ids"]) + } + } +} + +func TestMessageAttachmentJSONRejectsInvalidArrays(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + id := uuid.NewString() + for _, raw := range []string{ + "null", `"not an array"`, "{}", "1", "[null]", "[1]", `["bad"]`, + `["00000000-0000-0000-0000-000000000000"]`, + fmt.Sprintf("[%q]", strings.ReplaceAll(id, "-", "")), + fmt.Sprintf("[%q,%q]", id, strings.ToUpper(id)), + "[" + strings.TrimSuffix(strings.Repeat(fmt.Sprintf("%q,", id), 11), ",") + "]", + } { + t.Run(raw, func(t *testing.T) { + rec := sendJSON(r, c, "invalid", `{"text":"hello","attachment_ids":`+raw+"}") + if rec.Code != http.StatusBadRequest { + t.Fatalf("send: %d %s", rec.Code, rec.Body) + } + }) + } +} diff --git a/internal/chat/attachment_http_test.go b/internal/chat/attachment_http_test.go new file mode 100644 index 00000000..45510918 --- /dev/null +++ b/internal/chat/attachment_http_test.go @@ -0,0 +1,125 @@ +package chat_test + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], status, id, chatID, owner) + if err != nil { + t.Fatal(err) + } + return id +} + +func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + a, b := f.attachment(t, c, f.bob, "ready"), f.attachment(t, c, f.alice, "ready") + r := messageRouter(f, f.alice) + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a) + first := sendJSON(r, c, "attachments", body) + if first.Code != http.StatusCreated { + t.Fatalf("attachment-only send: %d %s", first.Code, first.Body) + } + var message chat.MessageResponse + if err := json.Unmarshal(first.Body.Bytes(), &message); err != nil { + t.Fatal(err) + } + if message.Text != "" || !reflect.DeepEqual(message.AttachmentIDs, []string{b.String(), a.String()}) { + t.Fatalf("message = %+v", message) + } + if again := sendJSON(r, c, "attachments", body); again.Code != http.StatusCreated || again.Body.String() != first.Body.String() { + t.Fatalf("replay: %d %s, want %s", again.Code, again.Body, first.Body) + } + for _, changed := range []string{ + fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, a, b), + fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, b), + fmt.Sprintf(`{"text":"different","attachment_ids":[%q,%q]}`, b, a), + } { + if rec := sendJSON(r, c, "attachments", changed); rec.Code != http.StatusConflict { + t.Fatalf("changed payload: %d %s", rec.Code, rec.Body) + } + } + rec := httptest.NewRecorder() + r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/chats/"+c.String()+"/messages", nil)) + var page pagination.PageResponse[chat.MessageResponse] + if rec.Code != http.StatusOK { + t.Fatalf("history: %d %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], message) { + t.Fatalf("history = %+v, want %+v", page, message) + } + without := sendJSON(r, c, "empty", `{"text":"text-only"}`) + empty := sendJSON(r, c, "empty", `{"attachment_ids":[],"text":"text-only"}`) + if without.Code != http.StatusCreated || empty.Code != http.StatusCreated || without.Body.String() != empty.Body.String() { + t.Fatalf("omitted vs empty replay: %d %s; %d %s", without.Code, without.Body, empty.Code, empty.Body) + } +} + +func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + other := f.newChat(t, f.alice) + ready := f.attachment(t, c, f.bob, "ready") + foreign := f.attachment(t, other, f.alice, "ready") + failed := f.attachment(t, c, f.alice, "failed") + pending := f.attachment(t, c, f.alice, "pending") + for _, tc := range []struct { + name string + actor, attachment uuid.UUID + want int + }{ + {"unknown", f.alice, ident.New(), http.StatusNotFound}, + {"foreign", f.alice, foreign, http.StatusNotFound}, + {"failed", f.alice, failed, http.StatusConflict}, + {"pending", f.alice, pending, http.StatusConflict}, + {"outsider", f.eve, ready, http.StatusNotFound}, + {"other-owner", f.alice, ready, http.StatusCreated}, + } { + t.Run(tc.name, func(t *testing.T) { + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment) + rec := sendJSON(messageRouter(f, tc.actor), c, "same", body) + if rec.Code != tc.want { + t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want) + } + }) + } +} + +func TestMessageJSONRequiresTextString(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for _, body := range []string{"null", `{}`, `{"text":null}`, `{"text":1}`, `{"text":[]}`, `{"text":"","attachment_ids":[]}`} { + if rec := sendJSON(r, c, "text", body); rec.Code != http.StatusBadRequest { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + } + if rec := sendJSON(r, c, "text", `{"text":"valid"}`); rec.Code != http.StatusCreated { + t.Fatalf("valid payload after rejected requests: %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/chat/attachment_message_test.go b/internal/chat/attachment_message_test.go new file mode 100644 index 00000000..d14678f8 --- /dev/null +++ b/internal/chat/attachment_message_test.go @@ -0,0 +1,66 @@ +package chat_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestAttachmentMessageValidatesAndOwnsOrderedIDs(t *testing.T) { + a, b := ident.New(), ident.New() + ids := []uuid.UUID{b, a} + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), "", ids...) + if err != nil || m.Text != "" || !reflect.DeepEqual(m.AttachmentIDs, ids) { + t.Fatalf("attachment-only message = %+v, %v", m, err) + } + ids[0] = a + if m.AttachmentIDs[0] != b { + t.Fatal("message aliases caller's attachment slice") + } + for _, tc := range []struct { + text string + ids []uuid.UUID + }{ + {"ok", []uuid.UUID{a, a}}, + {"ok", []uuid.UUID{uuid.Nil}}, + {"ok", make([]uuid.UUID, 11)}, + {strings.Repeat("x", 4001), []uuid.UUID{a}}, + {"nul\x00", []uuid.UUID{a}}, + } { + _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), tc.text, tc.ids...) + if err == nil || apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("invalid message accepted: %v", err) + } + } + ten := make([]uuid.UUID, 10) + for i := range ten { + ten[i] = ident.New() + } + if _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " \n", ten...); err != nil { + t.Fatalf("ten attachments: %v", err) + } +} + +func TestSameContentIncludesOrderedAttachmentsAndTreatsNilAsEmpty(t *testing.T) { + a, b := ident.New(), ident.New() + m := chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{b, a}} + for _, tc := range []struct { + ids []uuid.UUID + equal bool + }{ + {[]uuid.UUID{b, a}, true}, {[]uuid.UUID{a, b}, false}, {[]uuid.UUID{a}, false}, {nil, false}, + } { + if got := m.SameContent(chat.Message{Text: "hello", AttachmentIDs: tc.ids}); got != tc.equal { + t.Fatalf("attachments %v: equality=%v, want %v", tc.ids, got, tc.equal) + } + } + if !(chat.Message{Text: "hello"}).SameContent(chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{}}) { + t.Fatal("nil attachments must equal empty attachments") + } +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go index c913393e..95dc9089 100644 --- a/internal/chat/handler.go +++ b/internal/chat/handler.go @@ -1,6 +1,8 @@ package chat import ( + "bytes" + "encoding/json" "net/http" "regexp" @@ -36,7 +38,8 @@ type createChatRequest struct { } type createMessageRequest struct { - Text string `json:"text"` + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` } type ChatResponse struct { @@ -47,11 +50,12 @@ type ChatResponse struct { } type MessageResponse struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` } func toChatResponse(c ChatWithMembers) ChatResponse { @@ -68,12 +72,17 @@ func toChatResponse(c ChatWithMembers) ChatResponse { } func toMessageResponse(m Message) MessageResponse { + ids := make([]string, 0, len(m.AttachmentIDs)) + for _, id := range m.AttachmentIDs { + ids = append(ids, id.String()) + } return MessageResponse{ - ID: m.ID.String(), - ChatID: m.ChatID.String(), - SenderID: m.SenderID.String(), - Text: m.Text, - CreatedAt: httpx.Timestamp(m.CreatedAt), + AttachmentIDs: ids, + ID: m.ID.String(), + ChatID: m.ChatID.String(), + SenderID: m.SenderID.String(), + Text: m.Text, + CreatedAt: httpx.Timestamp(m.CreatedAt), } } @@ -144,7 +153,16 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } - message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text}) + if req.Text == nil { + httpx.Error(c, apperr.Invalid("text must be a string")) + return + } + attachments, err := decodeAttachmentIDs(req.AttachmentIDs) + if err != nil { + httpx.Error(c, err) + return + } + message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: *req.Text, AttachmentIDs: attachments}) if err != nil { httpx.Error(c, err) return @@ -207,3 +225,29 @@ func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) { } return limit, &last, nil } + +// RawMessage distinguishes an omitted optional array from an explicit null. +func decodeAttachmentIDs(raw json.RawMessage) ([]uuid.UUID, error) { + if raw == nil { + return nil, nil + } + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, apperr.Invalid("attachment_ids must be an array") + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil { + return nil, apperr.Invalid("attachment_ids must be an array of UUIDs") + } + if len(values) > MaxAttachments { + return nil, apperr.Invalid("attachment_ids must have at most 10 entries") + } + ids := make([]uuid.UUID, 0, len(values)) + for _, value := range values { + id, err := ident.Parse(value) + if err != nil { + return nil, apperr.Invalid("attachment_ids must contain valid UUIDs") + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/internal/chat/message.go b/internal/chat/message.go index 06a758bc..cda9b940 100644 --- a/internal/chat/message.go +++ b/internal/chat/message.go @@ -2,6 +2,7 @@ package chat import ( "fmt" + "slices" "strings" "time" "unicode/utf8" @@ -12,19 +13,36 @@ import ( "messenger/internal/platform/ident" ) -const MaxTextCodePoints = 4000 +const ( + MaxTextCodePoints = 4000 + MaxAttachments = 10 +) type Message struct { - ID uuid.UUID - ChatID uuid.UUID - SenderID uuid.UUID - Text string - CreatedAt time.Time + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + AttachmentIDs []uuid.UUID } -// NewMessage keeps the text verbatim; only blank or oversized text is rejected. -func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { - if strings.TrimSpace(text) == "" { +// NewMessage keeps text and attachment order verbatim; it owns its attachment slice. +func NewMessage(id, chatID, senderID uuid.UUID, text string, attachmentIDs ...uuid.UUID) (Message, error) { + if len(attachmentIDs) > MaxAttachments { + return Message{}, apperr.Invalid("attachment_ids must have at most 10 entries") + } + seen := make(map[uuid.UUID]struct{}, len(attachmentIDs)) + for _, id := range attachmentIDs { + if id == uuid.Nil { + return Message{}, apperr.Invalid("attachment_ids must contain non-nil UUIDs") + } + if _, duplicate := seen[id]; duplicate { + return Message{}, apperr.Invalid("attachment_ids must be unique") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 { return Message{}, apperr.Invalid("text must not be blank") } if utf8.RuneCountInString(text) > MaxTextCodePoints { @@ -35,16 +53,17 @@ func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { return Message{}, apperr.Invalid("text must not contain the NUL character") } return Message{ - ID: id, - ChatID: chatID, - SenderID: senderID, - Text: text, - CreatedAt: ident.TimeOf(id), + ID: id, + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: ident.TimeOf(id), + AttachmentIDs: slices.Clone(attachmentIDs), }, nil } // SameContent reports whether two messages say the same thing, whatever their // ids and times. func (m Message) SameContent(other Message) bool { - return m.Text == other.Text + return m.Text == other.Text && slices.Equal(m.AttachmentIDs, other.AttachmentIDs) } diff --git a/internal/chat/service.go b/internal/chat/service.go index ac71e96f..39a66f7c 100644 --- a/internal/chat/service.go +++ b/internal/chat/service.go @@ -78,15 +78,16 @@ type MessagePayload struct { ChatID uuid.UUID IdempotencyKey string Text string + AttachmentIDs []uuid.UUID } -// Send checks access and the text on every attempt, a replay included, as the +// Send checks access and content on every attempt, a replay included, as the // contract requires of a repeat. func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) { if err := s.requireMember(ctx, actor, p.ChatID); err != nil { return Message{}, err } - message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text) + message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text, p.AttachmentIDs...) if err != nil { return Message{}, err } diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go index 1098708c..c7a16681 100644 --- a/internal/chat/service_test.go +++ b/internal/chat/service_test.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "os" + "reflect" "strings" "sync/atomic" "testing" @@ -217,14 +218,14 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) { if err != nil { t.Fatal(err) } - if len(page.Items) != 1 || page.Items[0] != first || page.Next == nil || *page.Next != first.ID { + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], first) || page.Next == nil || *page.Next != first.ID { t.Fatalf("first page = %+v", page) } rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1) if err != nil { t.Fatal(err) } - if len(rest.Items) != 1 || rest.Items[0] != second || rest.Next != nil { + if len(rest.Items) != 1 || !reflect.DeepEqual(rest.Items[0], second) || rest.Next != nil { t.Fatalf("second page = %+v", rest) } if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) { diff --git a/internal/chat/store/attachments_test.go b/internal/chat/store/attachments_test.go new file mode 100644 index 00000000..ef4c3ac8 --- /dev/null +++ b/internal/chat/store/attachments_test.go @@ -0,0 +1,312 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, + hash[:], status, id, chatID, owner) + if err != nil { + t.Fatal(err) + } + return id +} + +func attachedMessage(t *testing.T, c, sender uuid.UUID, ids ...uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), c, sender, "", ids...) + if err != nil { + t.Fatal(err) + } + return m +} + +func assertCounts(t *testing.T, messages, keys, joins int) { + t.Helper() + var gotMessages, gotKeys, gotJoins int + err := db.Pool.QueryRow(context.Background(), ` + SELECT (SELECT count(*) FROM messages), + (SELECT count(*) FROM message_idempotency_keys), + (SELECT count(*) FROM message_attachments)`).Scan(&gotMessages, &gotKeys, &gotJoins) + if err != nil || gotMessages != messages || gotKeys != keys || gotJoins != joins { + t.Fatalf("counts = %d/%d/%d, %v; want %d/%d/%d", gotMessages, gotKeys, gotJoins, err, messages, keys, joins) + } +} + +func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + m := attachedMessage(t, c.ID, alice, ident.New()) + _, err := f.messages.AppendOnce(f.ctx, m, "unknown", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("unknown attachment: %v; want not found", err) + } + var messages, keys int + err = db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM messages), (SELECT count(*) FROM message_idempotency_keys)`).Scan(&messages, &keys) + if err != nil || messages != 0 || keys != 0 { + t.Fatalf("failed validation persisted %d messages/%d keys: %v", messages, keys, err) + } +} + +func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) { + for _, status := range []string{"foreign", "failed", "pending", "mixed"} { + t.Run(status, func(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + other := f.mustChat(t, alice) + ready := f.attachment(t, c.ID, bob, "ready") + var invalid uuid.UUID + want := apperr.KindConflict + switch status { + case "foreign": + invalid = f.attachment(t, other.ID, alice, "ready") + want = apperr.KindNotFound + case "mixed": + ready = f.attachment(t, c.ID, bob, "failed") + invalid = ident.New() + want = apperr.KindNotFound + default: + invalid = f.attachment(t, c.ID, alice, status) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ready, invalid), "bad", keyTTL) + if err == nil || apperr.KindOf(err) != want { + t.Fatalf("validation: %v; want %v", err, want) + } + assertCounts(t, 0, 0, 0) + good := f.attachment(t, c.ID, bob, "ready") + f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad") + assertCounts(t, 1, 1, 1) + }) + } +} + +func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + a, b := f.attachment(t, c.ID, bob, "ready"), f.attachment(t, c.ID, bob, "ready") + first := attachedMessage(t, c.ID, alice, b, a) + got := f.appendOnce(t, first, "k") + if !reflect.DeepEqual(got, first) { + t.Fatalf("append = %+v, want %+v", got, first) + } + replayed := f.appendOnce(t, attachedMessage(t, c.ID, alice, a, b), "k") + if !reflect.DeepEqual(replayed, first) { + t.Fatalf("replay must return original ordered attachments: %+v", replayed) + } + assertCounts(t, 1, 1, 2) + expire(t) + second := attachedMessage(t, c.ID, bob, a) + f.appendOnce(t, second, "k2") + replacement := attachedMessage(t, c.ID, alice, a, b) + f.appendOnce(t, replacement, "k") + assertCounts(t, 3, 2, 5) + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 1) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("first page = %+v, %v", page, err) + } + page, err = f.messages.ListByChat(f.ctx, c.ID, &first.ID, 10) + if err != nil || !reflect.DeepEqual(page, []chat.Message{second, replacement}) { + t.Fatalf("remaining page = %+v, %v", page, err) + } +} + +func TestAppendOnceRevalidatesAttachmentOnReplay(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "ready") + first := attachedMessage(t, c.ID, alice, a) + f.appendOnce(t, first, "k") + if _, err := db.Pool.Exec(f.ctx, `UPDATE files SET status = 'failed'`); err != nil { + t.Fatal(err) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("failed replay attachment = %v, want conflict", err) + } + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("failed blob must preserve history attachment ids: %+v, %v", page, err) + } + assertCounts(t, 1, 1, 1) +} + +func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "ready") + // A database fault at the join boundary must not leave a key or message. + _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments DROP CONSTRAINT test_reject_join`); err != nil { + t.Error(err) + } + }() + if _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL); err == nil { + t.Fatal("join fault must fail append") + } + assertCounts(t, 0, 0, 0) +} + +func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a, b := f.attachment(t, c.ID, alice, "ready"), f.attachment(t, c.ID, alice, "ready") + const requests = 20 + results := make(chan chat.Message, requests) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range requests { + ids := []uuid.UUID{a, b} + if i%2 != 0 { + ids = []uuid.UUID{b, a} + } + m := attachedMessage(t, c.ID, alice, ids...) + wg.Go(func() { + <-start + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + results <- got + }) + } + close(start) + wg.Wait() + close(results) + var winner *chat.Message + n := 0 + for got := range results { + n++ + if winner == nil { + winner = &got + } else if !reflect.DeepEqual(got, *winner) { + t.Fatalf("concurrent replay differs: %+v, want %+v", got, *winner) + } + } + if n != requests { + t.Fatalf("got %d successful replies, want %d", n, requests) + } + assertCounts(t, 1, 1, 2) +} + +type queryCounter struct{ count atomic.Int32 } + +func (c *queryCounter) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + c.count.Add(1) + return ctx +} +func (*queryCounter) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} + +func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + var ids []uuid.UUID + for range 10 { + ids = append(ids, f.attachment(t, c.ID, alice, "ready")) + } + cfg := db.Pool.Config() + counter := &queryCounter{} + cfg.ConnConfig.Tracer = counter + pool, err := pgxpool.NewWithConfig(f.ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := pool.Ping(f.ctx); err != nil { + t.Fatal(err) + } + repo := store.NewMessages(pool) + counter.count.Store(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids[0]), "one", keyTTL); err != nil { + t.Fatal(err) + } + one := counter.count.Swap(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids...), "ten", keyTTL); err != nil { + t.Fatal(err) + } + ten := counter.count.Swap(0) + if one != ten { + t.Fatalf("queries grew from %d to %d for 1 vs 10 attachments", one, ten) + } + page, err := repo.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 2 || !reflect.DeepEqual(page[1].AttachmentIDs, ids) { + t.Fatalf("history = %+v, %v", page, err) + } + if n := counter.count.Load(); n != 1 { + t.Fatalf("history used %d queries, want one", n) + } +} + +func TestAppendOnceRetriesWhenBoundKeyDisappears(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + f.appendOnce(t, first, "vanish") + next := f.newMessage(t, c.ID, alice) + // Simulate the missing-key boundary after an unsuccessful claim using a + // statement trigger. Its second invocation sees no key and is a no-op. + _, err := db.Pool.Exec(f.ctx, fmt.Sprintf(` + CREATE FUNCTION test_vanish_key() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM messages WHERE id = '%s'::uuid) THEN + DELETE FROM message_idempotency_keys WHERE key = 'vanish'; + END IF; + RETURN NULL; + END $$; + CREATE TRIGGER test_vanish_key AFTER INSERT ON messages + FOR EACH STATEMENT EXECUTE FUNCTION test_vanish_key()`, next.ID.String())) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `DROP TRIGGER test_vanish_key ON messages; DROP FUNCTION test_vanish_key()`); err != nil { + t.Error(err) + } + }() + ctx, cancel := context.WithTimeout(f.ctx, 3*time.Second) + defer cancel() + got, err := f.messages.AppendOnce(ctx, next, "vanish", keyTTL) + if err != nil || got.ID != next.ID { + t.Fatalf("missing bound key = %+v, %v; want new message", got, err) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index 6554bf9f..cd6df783 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -19,6 +19,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "messenger/internal/chat" + "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" ) @@ -118,9 +119,8 @@ func NewMessages(pool *pgxpool.Pool) *Messages { return &Messages{pool: pool} } -// claimSQL binds the key and inserts the message in one statement, so the -// key's lock never spans a round trip. An expired key is taken over; a valid -// one leaves claimed, and so the message insert, empty. +// claimSQL binds the key and inserts the message in one statement. An expired +// key is taken over; a valid one leaves claimed and the message insert empty. const claimSQL = ` WITH claimed AS ( INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) @@ -132,30 +132,100 @@ const claimSQL = ` INSERT INTO messages (chat_id, id, sender_id, text, created_at) SELECT $2, message_id, $1, $6, $7 FROM claimed` -// The bound message is read by a second statement: the claim's snapshot -// cannot see a key that a concurrent request has just committed. +// The transaction keeps attachment readiness, message joins and the key atomic. +// The replay uses a new statement snapshot to see a concurrent winner. func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { - tag, err := r.pool.Exec(ctx, claimSQL, - m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) - if err != nil { - return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) - } - if tag.RowsAffected() == 1 { - return m, nil - } var stored chat.Message - err = r.pool.QueryRow(ctx, ` - SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at - FROM message_idempotency_keys k - JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id - WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, - m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) + err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil { + return err + } + for { + if err := ctx.Err(); err != nil { + return err + } + tag, err := tx.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return fmt.Errorf("insert message: %w", err) + } + if tag.RowsAffected() == 1 { + if len(m.AttachmentIDs) > 0 { + _, err := tx.Exec(ctx, ` + INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal) + SELECT $1, $2, id, (ord - 1)::smallint + FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)`, + m.ChatID, m.ID, m.AttachmentIDs) + if err != nil { + return fmt.Errorf("insert message attachments: %w", err) + } + } + stored = m + return nil + } + stored, err = scanMessage(tx.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key)) + if errors.Is(err, pgx.ErrNoRows) { + // A vanished key is available again; retry its atomic claim. + continue + } + if err != nil { + return fmt.Errorf("select message by key: %w", err) + } + return nil + } + }) if err != nil { - return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) + return chat.Message{}, postgres.Classify(err) } return stored, nil } +func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids []uuid.UUID) error { + if len(ids) == 0 { + return nil + } + // Shared row locks prevent deletion or a readiness change until joins commit. + rows, err := tx.Query(ctx, ` + SELECT f.status + FROM attachments a + JOIN files f ON f.hash = a.file_hash + WHERE a.chat_id = $1 AND a.id = ANY($2::uuid[]) + ORDER BY a.id + FOR SHARE OF a, f`, chatID, ids) + if err != nil { + return fmt.Errorf("check attachments: %w", err) + } + statuses, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return fmt.Errorf("read attachment status: %w", err) + } + if len(statuses) != len(ids) { + return apperr.NotFound("attachment not found") + } + for _, status := range statuses { + if status != "ready" { + return apperr.Conflict("attachment is not ready") + } + } + return nil +} + +func scanMessage(row pgx.Row) (chat.Message, error) { + var m chat.Message + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs) + if len(m.AttachmentIDs) == 0 { + m.AttachmentIDs = nil + } + return m, err +} + // DeleteExpiredKeys skips rows locked by another cleaner or a request rather // than waiting for them. func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) { @@ -179,14 +249,16 @@ func interval(d time.Duration) string { return fmt.Sprintf("%d milliseconds", d.Milliseconds()) } -// ListByChat is the history query; scripts/explain-history.sql runs the same -// statement for the EXPLAIN evidence. +// ListByChat keeps keyset pagination on messages and collects each ordered +// attachment array in the same statement. func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) { rows, err := r.pool.Query(ctx, ` - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 - AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM messages m + WHERE m.chat_id = $1 + AND m.id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) ORDER BY id LIMIT $3`, chatID, after, limit) @@ -194,9 +266,7 @@ func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) } messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { - var m chat.Message - err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) - return m, err + return scanMessage(row) }) if err != nil { return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go index 8e843d23..d2b01270 100644 --- a/internal/chat/store/postgres_test.go +++ b/internal/chat/store/postgres_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "reflect" "sync" "testing" "time" @@ -214,7 +215,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got) != 3 || got[0] != first || got[1] != second || got[2] != third { + if len(got) != 3 || !reflect.DeepEqual(got[0], first) || !reflect.DeepEqual(got[1], second) || !reflect.DeepEqual(got[2], third) { t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got) } } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 936b8ddc..10b4b523 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -15,6 +15,9 @@ import ( "time" ) +// Allow a 30s binary request, its independent 3s cleanup, and 5s margin. +const defaultShutdownTimeout = 38 * time.Second + type Logging struct { Env string Level slog.Level @@ -43,6 +46,7 @@ type Config struct { IdempotencyTTL time.Duration DBMaxConns int32 DebugAddr string + S3 S3 } type Migrate struct { @@ -58,7 +62,7 @@ func Load(lookup Lookup) (Config, error) { cfg := Config{ Host: "0.0.0.0", Port: 8080, - ShutdownTimeout: 8 * time.Second, + ShutdownTimeout: defaultShutdownTimeout, MaxBodyBytes: 64 * 1024, SessionTTL: time.Hour, DrainDelay: 2 * time.Second, @@ -136,6 +140,11 @@ func Load(lookup Lookup) (Config, error) { if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { return Config{}, err } + if _, ok := lookup("S3_ENDPOINT"); ok { + if cfg.S3, err = LoadS3(lookup); err != nil { + return Config{}, err + } + } return cfg, nil } diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go index e58ffe19..914795b5 100644 --- a/internal/platform/config/config_test.go +++ b/internal/platform/config/config_test.go @@ -56,7 +56,7 @@ func TestDefaults(t *testing.T) { if cfg.Env != "lab" || cfg.Level != slog.LevelInfo { t.Fatalf("cfg = %+v", cfg) } - if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 { + if cfg.ShutdownTimeout != 38*time.Second || cfg.MaxBodyBytes != 65536 { t.Fatalf("cfg = %+v", cfg) } if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL { @@ -70,6 +70,19 @@ func TestDefaults(t *testing.T) { } } +func TestDefaultShutdownTimeoutCoversBinaryLifecycle(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatal(err) + } + // Binary requests can use 30s, followed by 3s independent cleanup. Keep + // another 5s for shutdown scheduling and closing the remaining resources. + budget := 30*time.Second + 3*time.Second + 5*time.Second + if cfg.ShutdownTimeout < budget { + t.Fatalf("default shutdown budget %v cannot cover binary request and cleanup budget %v", cfg.ShutdownTimeout, budget) + } +} + func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} { cases := map[string]func(map[string]string){ diff --git a/internal/platform/config/s3.go b/internal/platform/config/s3.go new file mode 100644 index 00000000..919abf8a --- /dev/null +++ b/internal/platform/config/s3.go @@ -0,0 +1,55 @@ +package config + +import ( + "fmt" + "strconv" +) + +type S3 struct { + Endpoint, Region, Bucket string + AccessKey, SecretKey string + MaxBytes, MaxPixels int64 +} + +func LoadS3(lookup Lookup) (S3, error) { + cfg := S3{Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", MaxBytes: 10485760, MaxPixels: 40000000} + for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} { + if v, ok := lookup(key); ok { + *dest = v + } + } + var err error + if cfg.AccessKey, err = secret(lookup, "S3_ACCESS_KEY_ID_FILE"); err != nil { + return S3{}, err + } + if cfg.SecretKey, err = secret(lookup, "S3_SECRET_ACCESS_KEY_FILE"); err != nil { + return S3{}, err + } + for _, p := range []struct { + key string + dest *int64 + max int64 + }{{"MAX_UPLOAD_BYTES", &cfg.MaxBytes, 10485760}, {"MAX_IMAGE_PIXELS", &cfg.MaxPixels, 40000000}} { + if v, ok := lookup(p.key); ok { + n, e := strconv.ParseInt(v, 10, 64) + if e != nil || n < 1 || n > p.max { + return S3{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max) + } + *p.dest = n + } + } + return cfg, nil +} + +// ReconcileS3 selects the separate listing identity used by background scans. +func ReconcileS3(lookup Lookup) Lookup { + return func(key string) (string, bool) { + if key == "S3_ACCESS_KEY_ID_FILE" { + return lookup("S3_RECONCILE_ACCESS_KEY_ID_FILE") + } + if key == "S3_SECRET_ACCESS_KEY_FILE" { + return lookup("S3_RECONCILE_SECRET_ACCESS_KEY_FILE") + } + return lookup(key) + } +} diff --git a/internal/platform/config/s3_test.go b/internal/platform/config/s3_test.go new file mode 100644 index 00000000..d05e9121 --- /dev/null +++ b/internal/platform/config/s3_test.go @@ -0,0 +1,27 @@ +package config_test + +import ( + "messenger/internal/platform/config" + "testing" +) + +func TestS3LimitsAndSecretFiles(t *testing.T) { + values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")} + cfg, err := config.LoadS3(env(values)) + if err != nil { + t.Fatal(err) + } + if cfg.AccessKey != "access" || cfg.SecretKey != "private" || cfg.MaxBytes != 10485760 || cfg.MaxPixels != 40000000 { + t.Fatal("S3 defaults or secret trimming broken") + } + for _, tc := range []struct{ k, v string }{{"MAX_UPLOAD_BYTES", "0"}, {"MAX_UPLOAD_BYTES", "10485761"}, {"MAX_IMAGE_PIXELS", "40000001"}, {"MAX_IMAGE_PIXELS", "x"}} { + v := map[string]string{} + for k, x := range values { + v[k] = x + } + v[tc.k] = tc.v + if _, err := config.LoadS3(env(v)); err == nil { + t.Fatalf("accepted %s=%s", tc.k, tc.v) + } + } +} diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index b4844e9d..052e0d2d 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -123,6 +123,11 @@ func Timeout(d time.Duration) gin.HandlerFunc { // decoder as *http.MaxBytesError. func bodyLimit(maxBytes int64) gin.HandlerFunc { return func(c *gin.Context) { + // The authenticated binary handler installs its own 10 MiB limit. + if c.Request.Method == http.MethodPost && c.FullPath() == "/api/v1/chats/:chat_id/attachments" { + c.Next() + return + } if c.Request.Body != nil { c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) } diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go new file mode 100644 index 00000000..7a6385e9 --- /dev/null +++ b/internal/platform/s3store/bootstrap_test.go @@ -0,0 +1,142 @@ +package s3store_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/s3store" +) + +func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) { + var ready atomic.Bool + var reads atomic.Int32 + var mu sync.Mutex + objects := map[string]string{".health/storage-v1": "messenger s3 readiness v1\n"} + var freshReads, deleted int + requested := make(chan struct{}, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + if r.Method == "GET" && key == ".health/storage-v1" { + reads.Add(1) + if !ready.Load() { + select { + case requested <- struct{}{}: + default: + } + w.WriteHeader(500) + io.WriteString(w, `InternalErrorvolume not registered`) + return + } + } + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "GET": + value, ok := objects[key] + if !ok { + w.WriteHeader(404) + io.WriteString(w, `NoSuchKey`) + return + } + if key != ".health/storage-v1" { + freshReads++ + } + w.Header().Set("ETag", `"probe"`) + w.Header().Set("Last-Modified", "Mon, 16 Nov 2026 08:00:00 GMT") + w.Header().Set("Content-Length", fmt.Sprint(len(value))) + io.WriteString(w, value) + case "PUT": + var body io.Reader = r.Body + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") { + body = httputil.NewChunkedReader(body) + } + data, err := io.ReadAll(io.LimitReader(body, 1024)) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + objects[key] = string(data) + w.Header().Set("ETag", `"probe"`) + case "DELETE": + if key == ".health/storage-v1" { + t.Error("deleted retained readiness marker") + } + delete(objects, key) + deleted++ + w.WriteHeader(204) + default: + t.Errorf("unexpected storage probe: %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer server.Close() + s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- s.Bootstrap(ctx) }() + select { + case err := <-done: + t.Fatalf("bootstrap returned before reading retained storage: %v", err) + case <-requested: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case err := <-done: + t.Fatalf("bootstrap accepted unavailable retained storage: %v", err) + case <-time.After(40 * time.Millisecond): + } + ready.Store(true) + if err := <-done; err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if reads.Load() < 2 || freshReads == 0 || deleted == 0 || len(objects) != 1 { + t.Fatalf("incomplete readiness proof: retainedReads=%d freshReads=%d deletes=%d remaining=%d", reads.Load(), freshReads, deleted, len(objects)) + } +} + +func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + w.WriteHeader(503) + io.WriteString(w, `ServiceUnavailablevolume not registered`) + })) + defer server.Close() + s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + if err := s.Bootstrap(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("storage readiness escaped its deadline: %v", err) + } + if time.Since(started) > time.Second { + t.Fatal("readiness did not honor caller deadline") + } +} diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go new file mode 100644 index 00000000..9798a581 --- /dev/null +++ b/internal/platform/s3store/integration_test.go @@ -0,0 +1,270 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "messenger/internal/attachment" + "messenger/internal/platform/s3store" + "messenger/internal/testenv" +) + +func TestSeaweedStore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + newClient := func(access, secret string) *minio.Client { + t.Helper() + c, err := minio.New(strings.TrimPrefix(s.Endpoint, "http://"), &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Region: s.Region, BucketLookup: minio.BucketLookupPath, MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return c + } + api := newClient(s.APIAccessKey, s.APISecretKey) + admin := newClient(s.AdminAccessKey, s.AdminSecretKey) + denied := func(t *testing.T, err error) { + t.Helper() + var response minio.ErrorResponse + if !errors.As(err, &response) || response.StatusCode != http.StatusForbidden { + t.Fatalf("expected access denied, got %v", err) + } + } + t.Run("remote gRPC cannot modify identity cache", func(t *testing.T) { + conn, err := grpc.NewClient("passthrough:///"+s.GRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) { + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, "tcp", address) + })) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + opCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + err = conn.Invoke(opCtx, "/messaging_pb.SeaweedS3IamCache/PutIdentity", &emptypb.Empty{}, &emptypb.Empty{}) + if code := status.Code(err); code != codes.PermissionDenied && code != codes.Unauthenticated { + t.Fatalf("remote IAM mutation was not rejected before argument validation: %v", err) + } + }) + t.Run("bootstrap is idempotent and API cannot administer", func(t *testing.T) { + if err := s.Admin.Bootstrap(ctx); err != nil { + t.Fatal(err) + } + denied(t, s.APIStore.Bootstrap(ctx)) + denied(t, api.MakeBucket(ctx, "forbidden-bucket", minio.MakeBucketOptions{Region: s.Region})) + denied(t, api.RemoveBucket(ctx, s.Bucket)) + buckets, err := api.ListBuckets(ctx) + // SeaweedFS returns a filtered empty result for an authenticated caller + // without List rights, rather than rejecting ListBuckets outright. + if err != nil || len(buckets) != 0 { + t.Fatalf("API enumerated buckets: count=%d error=%v", len(buckets), err) + } + _, err = s.APIStore.List(ctx, "", "", 1) + denied(t, err) + }) + t.Run("known length nonseekable put copy stat and stream", func(t *testing.T) { + const payload = "nonseekable image payload" + r := struct{ io.Reader }{strings.NewReader(payload)} + started := time.Now() + if err := s.APIStore.Put(ctx, "staging/source", r, int64(len(payload)), "image/png"); err != nil { + t.Fatal(err) + } + t.Logf("cold first PUT: %s", time.Since(started)) + if err := s.APIStore.Copy(ctx, "staging/source", "blobs/durable"); err != nil { + t.Fatal(err) + } + obj, err := s.APIStore.Stat(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + if obj.Key != "blobs/durable" || obj.Size != 25 || obj.Modified.IsZero() { + t.Fatalf("wrong metadata: %+v", obj) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != payload { + t.Fatalf("wrong body: %q", b) + } + meta, err := api.StatObject(ctx, s.Bucket, "blobs/durable", minio.StatObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if meta.ContentType != "image/png" { + t.Fatalf("copy lost MIME: %s", meta.ContentType) + } + for range 2 { + if err := s.APIStore.Delete(ctx, "staging/source"); err != nil { + t.Fatal(err) + } + } + if _, err := s.APIStore.Stat(ctx, "staging/source"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("API delete did not remove staging: %v", err) + } + }) + t.Run("anonymous get put head and listing are refused", func(t *testing.T) { + client := &http.Client{Timeout: 3 * time.Second} + defer client.CloseIdleConnections() + for _, tc := range []struct{ method, path string }{{"GET", "/"}, {"GET", "/" + s.Bucket + "?list-type=2"}, {"GET", "/" + s.Bucket + "/blobs/durable"}, {"HEAD", "/" + s.Bucket + "/blobs/durable"}, {"PUT", "/" + s.Bucket + "/anonymous"}} { + req, _ := http.NewRequestWithContext(ctx, tc.method, s.Endpoint+tc.path, strings.NewReader("x")) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("anonymous %s %s: %d", tc.method, tc.path, resp.StatusCode) + } + } + }) + t.Run("identities cannot escape their bucket", func(t *testing.T) { + if err := admin.MakeBucket(ctx, "private-other", minio.MakeBucketOptions{Region: s.Region}); err != nil { + t.Fatal(err) + } + if _, err := admin.PutObject(ctx, "private-other", "secret", strings.NewReader("private"), 7, minio.PutObjectOptions{DisableMultipart: true}); err != nil { + t.Fatal(err) + } + for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} { + outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1]) + if err != nil { + t.Fatal(err) + } + _, err = outside.Stat(ctx, "secret") + denied(t, err) + _, err = outside.Open(ctx, "secret") + denied(t, err) + denied(t, outside.Put(ctx, "secret", strings.NewReader("x"), 1, "image/png")) + denied(t, outside.Delete(ctx, "secret")) + _, err = outside.List(ctx, "", "", 1) + denied(t, err) + c := newClient(pair[0], pair[1]) + _, err = c.CopyObject(ctx, minio.CopyDestOptions{Bucket: s.Bucket, Object: "blobs/stolen"}, minio.CopySrcOptions{Bucket: "private-other", Object: "secret"}) + denied(t, err) + } + }) + t.Run("exclusive lexicographic pages including absent cursor", func(t *testing.T) { + for _, key := range []string{"page/z", "page/a", "page/a/nested", "page/b", "page/c"} { + if err := s.Store.Put(ctx, key, strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + } + var got []string + after := "" + for { + page, err := s.Store.List(ctx, "page/", after, 2) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + for _, obj := range page { + if obj.Size != 1 || obj.Modified.IsZero() { + t.Fatalf("wrong list metadata: %+v", obj) + } + got = append(got, obj.Key) + } + after = page[len(page)-1].Key + if len(got) > 5 { + t.Fatal("pagination did not advance") + } + } + if strings.Join(got, ",") != "page/a,page/a/nested,page/b,page/c,page/z" { + t.Fatalf("wrong order: %v", got) + } + page, err := s.Store.List(ctx, "page/", "page/az", 2) + if err != nil { + t.Fatal(err) + } + if len(page) != 2 || page[0].Key != "page/b" || page[1].Key != "page/c" { + t.Fatalf("absent cursor: %+v", page) + } + }) + t.Run("missing objects differ from bucket and permission failures", func(t *testing.T) { + if _, err := s.Store.Stat(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("stat missing: %v", err) + } + if _, err := s.Store.Open(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("open missing: %v", err) + } + if err := s.Store.Copy(ctx, "absent", "blobs/absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("copy missing: %v", err) + } + for range 2 { + if err := s.Store.Delete(ctx, "absent"); err != nil { + t.Fatal(err) + } + } + missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey) + if err != nil { + t.Fatal(err) + } + if _, err := missingBucket.Stat(ctx, "absent"); err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("missing bucket treated as missing object: %v", err) + } + }) + t.Run("short readers are not successful uploads", func(t *testing.T) { + if err := s.Store.Put(ctx, "staging/short", struct{ io.Reader }{strings.NewReader("short")}, 100, "image/png"); err == nil { + t.Fatal("short body accepted") + } + }) + t.Run("container replacement preserves volume data and authentication", func(t *testing.T) { + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != "nonseekable image payload" { + t.Fatalf("object lost on replacement: %q", b) + } + if _, err := s.Store.List(ctx, "page/", "", 2); err != nil { + t.Fatal(err) + } + }) + t.Run("outage is bounded and never missing", func(t *testing.T) { + if err := s.Stop(ctx); err != nil { + t.Fatal(err) + } + opCtx, stop := context.WithTimeout(ctx, 150*time.Millisecond) + defer stop() + start := time.Now() + _, err := s.Store.Stat(opCtx, "blobs/durable") + if err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("outage treated as missing: %v", err) + } + if time.Since(start) > 2*time.Second { + t.Fatal("outage exceeded caller deadline") + } + }) +} diff --git a/internal/platform/s3store/readiness.go b/internal/platform/s3store/readiness.go new file mode 100644 index 00000000..1f6830e3 --- /dev/null +++ b/internal/platform/s3store/readiness.go @@ -0,0 +1,114 @@ +package s3store + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/minio/minio-go/v7" + + "messenger/internal/attachment" +) + +const ( + storageMarkerKey = ".health/storage-v1" + storageMarkerBody = "messenger s3 readiness v1\n" +) + +var errStorageMarker = errors.New("s3: storage readiness marker has unexpected contents") + +// WaitReady is a startup barrier, deliberately separate from API readiness and +// ordinary object operations. Gateway authentication and bucket metadata can be +// available before the master elects its leader and registers retained volumes. +// Read the persistent marker before writing anything, then prove new writes, +// reads and deletes work. The marker lives outside the reconciler's namespaces. +func (s *Store) WaitReady(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + readyCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + probeKey := ".health/check-" + rand.Text() + dirty := false + defer func() { + if dirty { + cleanupCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + defer stop() + _ = s.Delete(cleanupCtx, probeKey) + } + }() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + err := s.checkStorage(readyCtx, probeKey, &dirty) + if err == nil { + return nil + } + if permanentStorageError(err) { + return err + } + select { + case <-readyCtx.Done(): + return fmt.Errorf("s3: wait for readable and writable storage: %w (last probe: %v)", readyCtx.Err(), err) + case <-ticker.C: + } + } +} + +func (s *Store) checkStorage(ctx context.Context, probeKey string, dirty *bool) error { + err := s.readStorageMarker(ctx, storageMarkerKey) + if errors.Is(err, attachment.ErrObjectMissing) { + // Only a genuinely absent marker permits its first creation. Never + // overwrite retained data to hide an unavailable volume or a bad marker. + if err := s.Put(ctx, storageMarkerKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + err = s.readStorageMarker(ctx, storageMarkerKey) + } + if err != nil { + return err + } + *dirty = true + if err := s.Put(ctx, probeKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + if err := s.readStorageMarker(ctx, probeKey); err != nil { + return err + } + if err := s.Delete(ctx, probeKey); err != nil { + return err + } + *dirty = false + return nil +} + +func (s *Store) readStorageMarker(ctx context.Context, key string) error { + r, err := s.Open(ctx, key) + if err != nil { + return err + } + data, readErr := io.ReadAll(io.LimitReader(r, int64(len(storageMarkerBody))+1)) + closeErr := r.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + if string(data) != storageMarkerBody { + return errStorageMarker + } + return nil +} + +func permanentStorageError(err error) bool { + if errors.Is(err, errStorageMarker) { + return true + } + var response minio.ErrorResponse + return errors.As(err, &response) && response.StatusCode >= 400 && response.StatusCode < 500 && response.StatusCode != 408 && response.StatusCode != 429 +} diff --git a/internal/platform/s3store/readiness_test.go b/internal/platform/s3store/readiness_test.go new file mode 100644 index 00000000..c92c5fb2 --- /dev/null +++ b/internal/platform/s3store/readiness_test.go @@ -0,0 +1,48 @@ +package s3store_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "messenger/internal/testenv" +) + +func TestRecreateReturnsWithRetainedObjectReadable(t *testing.T) { + // Three independent server replacements each have the production 60-second + // startup budget; no per-request transport timeout is changed by this test. + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + defer func() { + if t.Failed() { + t.Log(s.Diagnostics(context.Background())) + } + }() + const payload = "retained storage readiness proof" + if err := s.Store.Put(ctx, "blobs/retained", strings.NewReader(payload), int64(len(payload)), "application/octet-stream"); err != nil { + t.Fatal(err) + } + for attempt := range 3 { + started := time.Now() + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + r, err := s.Store.Open(ctx, "blobs/retained") + if err != nil { + t.Fatalf("replacement %d returned before retained bytes were readable (%s): %v", attempt+1, time.Since(started), err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil || string(data) != payload { + t.Fatalf("replacement %d lost retained bytes: %q %v", attempt+1, data, err) + } + t.Logf("replacement %d ready and retained bytes verified in %s", attempt+1, time.Since(started)) + } +} diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go new file mode 100644 index 00000000..2c4b9b9e --- /dev/null +++ b/internal/platform/s3store/store.go @@ -0,0 +1,198 @@ +package s3store + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + + "messenger/internal/attachment" +) + +const MaxPageSize = 500 + +type Store struct { + client *minio.Core + region string + bucket string +} + +var _ attachment.Blobs = (*Store)(nil) + +// New validates configuration without contacting S3, so text-only service startup +// and readiness do not depend on object storage being reachable. +func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) { + u, err := url.Parse(endpoint) + if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment") + } + if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" { + return nil, errors.New("s3: region and credentials are required") + } + if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil { + return nil, fmt.Errorf("s3: invalid bucket: %w", err) + } + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: time.Second, KeepAlive: 30 * time.Second}).DialContext, + TLSHandshakeTimeout: time.Second, + // The first write can wait for the single master's election and volume + // creation. The caller's shorter context still takes precedence. + ResponseHeaderTimeout: 30 * time.Second, + ExpectContinueTimeout: time.Second, + IdleConnTimeout: 30 * time.Second, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 32, + MaxConnsPerHost: 64, + DisableCompression: true, + } + client, err := minio.NewCore(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: u.Scheme == "https", + Region: region, + BucketLookup: minio.BucketLookupPath, + Transport: transport, + // A consumed upload stream cannot be replayed. Let the caller decide + // whether to retry a whole operation within its own deadline. + MaxRetries: 1, + }) + if err != nil { + return nil, fmt.Errorf("s3: client: %w", err) + } + return &Store{client: client, region: region, bucket: bucket}, nil +} + +// Bootstrap is used only by the admin init command, never by API startup. +func (s *Store) Bootstrap(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region}) + code := minio.ToErrorResponse(err).Code + if err != nil && code != "BucketAlreadyOwnedByYou" { + return wrap("create bucket", err) + } + return s.WaitReady(ctx) +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + if err := ctx.Err(); err != nil { + return err + } + if size < 0 || r == nil { + return errors.New("s3: PUT requires a reader and known nonnegative length") + } + _, err := s.client.PutObject(ctx, s.bucket, key, r, size, "", "", minio.PutObjectOptions{ + ContentType: mime, + DisableMultipart: true, + }) + return wrap("put object", err) +} + +func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Core.GetObject performs GET eagerly and returns the streaming body; + // Client.GetObject is lazy and would defer missing/access errors to Read. + r, _, _, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, wrap("open object", err) + } + return r, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (attachment.Object, error) { + if err := ctx.Err(); err != nil { + return attachment.Object{}, err + } + obj, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + // HEAD has no XML error body. The SDK synthesizes NoSuchKey for any + // object-path 404, including a missing bucket; confirm the bucket first. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + exists, bucketErr := s.client.BucketExists(ctx, s.bucket) + if bucketErr != nil { + return attachment.Object{}, wrap("stat bucket", bucketErr) + } + if !exists { + return attachment.Object{}, errors.New("s3: stat object: bucket does not exist") + } + } + return attachment.Object{}, wrap("stat object", err) + } + return attachment.Object{Key: key, Size: obj.Size, Modified: obj.LastModified}, nil +} + +func (s *Store) Copy(ctx context.Context, source, dest string) error { + if err := ctx.Err(); err != nil { + return err + } + _, err := s.client.Client.CopyObject(ctx, + minio.CopyDestOptions{Bucket: s.bucket, Object: dest}, + minio.CopySrcOptions{Bucket: s.bucket, Object: source}, + ) + return wrap("copy object", err) +} + +func (s *Store) Delete(ctx context.Context, key string) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) + // S3 DELETE is idempotent even when the object no longer exists. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return wrap("delete object", err) +} + +// List returns at most limit keys in byte-lexicographic order, strictly after +// after. The synchronous iterator stops before fetching the next page. +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit < 1 || limit > MaxPageSize { + return nil, fmt.Errorf("s3: list limit must be between 1 and %d", MaxPageSize) + } + listCtx, cancel := context.WithCancel(ctx) + defer cancel() + objects := make([]attachment.Object, 0, limit) + for obj := range s.client.ListObjectsIter(listCtx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, StartAfter: after, Recursive: true, MaxKeys: limit, + }) { + if obj.Err != nil { + return nil, wrap("list objects", obj.Err) + } + objects = append(objects, attachment.Object{Key: obj.Key, Size: obj.Size, Modified: obj.LastModified}) + if len(objects) == limit { + break + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + return objects, nil +} + +func wrap(op string, err error) error { + if err == nil { + return nil + } + // Never treat authorization errors, a missing bucket, or an outage as a + // missing blob: the reconciler would otherwise mark valid files failed. + if code := minio.ToErrorResponse(err).Code; code == "NoSuchKey" || code == "NoSuchObject" { + return fmt.Errorf("s3: %s: %w", op, attachment.ErrObjectMissing) + } + return fmt.Errorf("s3: %s: %w", op, err) +} diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go new file mode 100644 index 00000000..cac6af56 --- /dev/null +++ b/internal/platform/s3store/store_test.go @@ -0,0 +1,171 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/s3store" +) + +func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) })) + defer server.Close() + if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatal("constructor performed network I/O") + } + for _, tc := range []struct{ name, endpoint, region, bucket, access, secret string }{ + {"scheme", "ftp://localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"path", "http://localhost:8333/path", "us-east-1", "attachments", "access", "secret"}, + {"userinfo", "http://user:password@localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"query", "http://localhost:8333?x=1", "us-east-1", "attachments", "access", "secret"}, + {"fragment", "http://localhost:8333/#x", "us-east-1", "attachments", "access", "secret"}, + {"region", server.URL, "", "attachments", "access", "secret"}, + {"bucket", server.URL, "us-east-1", "INVALID", "access", "secret"}, + {"access", server.URL, "us-east-1", "attachments", "", "secret"}, + {"secret", server.URL, "us-east-1", "attachments", "access", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil { + t.Fatal("accepted invalid configuration") + } + }) + } +} + +func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) { + first := make(chan struct{}) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodPut || r.URL.Path != "/attachments/staging/stream" || r.URL.RawQuery != "" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(400) + return + } + if r.ContentLength <= 0 || len(r.TransferEncoding) != 0 { + t.Error("PUT is not a known-length single request") + } + b := make([]byte, 1) + if _, err := io.ReadFull(r.Body, b); err != nil { + t.Error(err) + return + } + close(first) + if _, err := io.Copy(io.Discard, r.Body); err != nil { + t.Error(err) + return + } + w.Header().Set("ETag", `"probe"`) + })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // One signing chunk can be buffered; the remaining bytes wait for the server. + r, w := io.Pipe() + defer r.Close() + go func() { + defer w.Close() + if _, err := w.Write(make([]byte, 256*1024)); err != nil { + return + } + select { + case <-first: + case <-ctx.Done(): + _ = w.CloseWithError(ctx.Err()) + return + } + _, _ = w.Write(make([]byte, 256*1024)) + }() + if err := store.Put(ctx, "staging/stream", r, 512*1024, "image/png"); err != nil { + t.Fatal(err) + } + if requests.Load() != 1 { + t.Fatalf("got %d requests, want one PUT", requests.Load()) + } +} + +func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for name, call := range map[string]func() error{ + "put": func() error { return store.Put(ctx, "key", strings.NewReader("x"), 1, "image/png") }, + "open": func() error { + r, e := store.Open(ctx, "key") + if r != nil { + r.Close() + } + return e + }, + "stat": func() error { _, e := store.Stat(ctx, "key"); return e }, + "copy": func() error { return store.Copy(ctx, "a", "b") }, + "delete": func() error { return store.Delete(ctx, "key") }, + "list": func() error { _, e := store.List(ctx, "", "", 1); return e }, + "bootstrap": func() error { return store.Bootstrap(ctx) }, + } { + t.Run(name, func(t *testing.T) { + if err := call(); !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want canceled", err) + } + }) + } + if err := store.Put(context.Background(), "key", strings.NewReader("x"), -1, "image/png"); err == nil { + t.Fatal("accepted unknown length") + } + for _, limit := range []int{-1, 0, 501, 1000000000} { + if _, err := store.List(context.Background(), "", "", limit); err == nil { + t.Fatalf("accepted unbounded page %d", limit) + } + } + if requests.Load() != 0 { + t.Fatalf("invalid calls reached server %d times", requests.Load()) + } +} + +func TestListStopsAfterOneBoundedPage(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Query().Get("start-after") != "blobs/a" || r.URL.Query().Get("prefix") != "blobs/" || r.URL.Query().Get("max-keys") != "2" { + t.Errorf("wrong list query: %s", r.URL.RawQuery) + } + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`) + })) + defer server.Close() + store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + if err != nil { + t.Fatal(err) + } + objects, err := store.List(context.Background(), "blobs/", "blobs/a", 2) + if err != nil { + t.Fatal(err) + } + if len(objects) != 2 || objects[0].Key != "blobs/b" || objects[1].Key != "blobs/c" { + t.Fatalf("wrong page: %+v", objects) + } + if requests.Load() != 1 { + t.Fatalf("prefetched %d pages", requests.Load()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 64f7abf8..661c5e10 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -24,8 +24,8 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, srv := &http.Server{ Handler: h, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + ReadTimeout: 35 * time.Second, + WriteTimeout: 35 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 64 * 1024, ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 8a2caa39..c8685c72 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,21 +1,198 @@ package server_test import ( + "bytes" "context" + "fmt" "io" "log/slog" "net" "net/http" + "os" + "path/filepath" + "strconv" + "sync" "testing" "time" "github.com/gin-gonic/gin" "messenger/internal/health" + "messenger/internal/platform/config" "messenger/internal/platform/httpx" "messenger/internal/server" ) +func serverDefaultConfig(t *testing.T) config.Config { + t.Helper() + values := make(map[string]string) + for key, content := range map[string]string{ + "DATABASE_URL_FILE": "postgres://unused:unused@localhost:5432/unused", + "SESSION_STORE_URL_FILE": "redis://localhost:6379/0", + } { + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + values[key] = path + } + cfg, err := config.Load(func(key string) (string, bool) { value, ok := values[key]; return value, ok }) + if err != nil { + t.Fatal(err) + } + return cfg +} + +func TestDefaultShutdownAllowsActiveBinaryTransfers(t *testing.T) { + for _, direction := range []string{"upload", "download"} { + t.Run(direction, func(t *testing.T) { + t.Parallel() + cfg := serverDefaultConfig(t) + payload := []byte("binary\x00\xff\x01request-and-response\x00\xfe") + prefix := payload[:3] + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + bodyReader, bodyWriter := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + requestCtx, cancelRequest := context.WithTimeout(context.Background(), 25*time.Second) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + probes := health.New() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + if direction == "upload" { + first := make([]byte, len(prefix)) + if _, err := io.ReadFull(r.Body, first); err != nil { + return + } + close(started) + rest, err := io.ReadAll(r.Body) + if err != nil || !bytes.Equal(append(first, rest...), payload) { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = w.Write(payload) + return + } + _, _ = w.Write(prefix) + w.(http.Flusher).Flush() + close(started) + select { + case <-release: + _, _ = w.Write(payload[len(prefix):]) + case <-r.Context().Done(): + } + }) + stopped, exited := make(chan error, 1), make(chan struct{}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + go func() { + defer close(exited) + stopped <- server.Run(ctx, log, ln, handler, probes, cfg.DrainDelay, cfg.ShutdownTimeout) + }() + t.Cleanup(func() { + cancel() + cancelRequest() + unblock() + _ = bodyReader.Close() + _ = bodyWriter.Close() + _ = ln.Close() + select { + case <-exited: + case <-time.After(3 * time.Second): + t.Error("server did not stop after transfer cleanup") + } + }) + client := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + defer client.CloseIdleConnections() + method := http.MethodGet + var body io.Reader + if direction == "upload" { + method = http.MethodPut + body = bodyReader + } + req, err := http.NewRequestWithContext(requestCtx, method, "http://"+ln.Addr().String()+"/binary", body) + if err != nil { + t.Fatal(err) + } + if direction == "upload" { + req.ContentLength = int64(len(payload)) + req.Header.Set("Content-Type", "application/octet-stream") + } + completed := make(chan error, 1) + go func() { + resp, err := client.Do(req) + if err != nil { + completed <- err + return + } + defer resp.Body.Close() + got, err := io.ReadAll(resp.Body) + if err == nil && (resp.StatusCode != http.StatusOK || !bytes.Equal(got, payload)) { + err = fmt.Errorf("binary response status=%d body=%q, want exact payload", resp.StatusCode, got) + } + completed <- err + }() + if direction == "upload" { + if _, err := bodyWriter.Write(prefix); err != nil { + t.Fatal(err) + } + } + select { + case <-started: + case err := <-completed: + t.Fatalf("transfer failed before shutdown: %v", err) + case <-requestCtx.Done(): + t.Fatal("transfer never became active") + } + cancel() + // Nine seconds inside Shutdown exceeds the former 8s budget. Include + // the configured drain phase so it cannot hide a premature force-close. + timer := time.NewTimer(cfg.DrainDelay + 9*time.Second) + defer timer.Stop() + select { + case err := <-stopped: + t.Fatalf("shutdown interrupted active %s before binary transfer completed: %v", direction, err) + case err := <-completed: + t.Fatalf("active %s ended before remaining bytes were released: %v", direction, err) + case <-timer.C: + } + if direction == "upload" { + if _, err := bodyWriter.Write(payload[len(prefix):]); err != nil { + t.Fatal(err) + } + if err := bodyWriter.Close(); err != nil { + t.Fatal(err) + } + } else { + unblock() + } + select { + case err := <-completed: + if err != nil { + t.Fatalf("active %s was truncated during shutdown: %v", direction, err) + } + case <-requestCtx.Done(): + t.Fatal("binary transfer did not finish after release") + } + select { + case err := <-stopped: + if err != nil { + t.Fatalf("graceful shutdown: %v", err) + } + case <-requestCtx.Done(): + t.Fatal("server did not finish graceful shutdown") + } + if probes.IsReady() { + t.Fatal("readiness remained enabled after shutdown") + } + }) + } +} + // newHandler builds the smallest handler with probes; the lifecycle does not // depend on the business modules or their stores. func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go new file mode 100644 index 00000000..062c9f99 --- /dev/null +++ b/internal/testenv/s3.go @@ -0,0 +1,206 @@ +package testenv + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/google/uuid" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "messenger/internal/platform/s3store" +) + +const S3Image = "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0" + +// S3 keeps independent API, reconciler and bootstrap identities. Store uses the +// reconciler identity so tests can inspect and clean up their objects. +type S3 struct { + Store, APIStore, Admin *s3store.Store + Endpoint, Region, Bucket string + GRPCAddress string + AccessKey, SecretKey string + APIAccessKey, APISecretKey string + AdminAccessKey, AdminSecretKey string + container testcontainers.Container + volume string + identities, options []byte +} + +func StartS3(ctx context.Context) (*S3, error) { + useDockerDesktopPipe() + s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab4-s3-test-" + uuid.NewString()} + keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey} + for _, key := range keys { + b := make([]byte, 24) + if _, err := rand.Read(b); err != nil { + return nil, err + } + *key = hex.EncodeToString(b) + } + type credential struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + } + type identity struct { + Name string `json:"name"` + Credentials []credential `json:"credentials"` + Actions []string `json:"actions"` + } + config := struct { + Identities []identity `json:"identities"` + }{Identities: []identity{ + {"bootstrap", []credential{{s.AdminAccessKey, s.AdminSecretKey}}, []string{"Admin", "Read", "Write", "List"}}, + {"api", []credential{{s.APIAccessKey, s.APISecretKey}}, []string{"Read:" + s.Bucket, "Write:" + s.Bucket}}, + {"reconciler", []credential{{s.AccessKey, s.SecretKey}}, []string{"List:" + s.Bucket, "Read:" + s.Bucket, "Write:" + s.Bucket}}, + }} + var err error + s.identities, err = json.Marshal(config) + if err != nil { + return nil, err + } + _, source, _, _ := runtime.Caller(0) + s.options, err = os.ReadFile(filepath.Join(filepath.Dir(source), "../../infra/s3/seaweedfs.options")) + if err != nil { + return nil, fmt.Errorf("read S3 options: %w", err) + } + startCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + if err := s.launch(startCtx); err != nil { + s.Close() + return nil, err + } + if err := s.Admin.Bootstrap(startCtx); err != nil { + s.Close() + return nil, err + } + return s, nil +} + +func (s *S3) launch(ctx context.Context) error { + pids := int64(256) + firstStatus := 0 + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: S3Image, + Cmd: []string{"server", "-options=/etc/seaweedfs/lab.options"}, + Env: map[string]string{"GOMEMLIMIT": "460MiB"}, + ExposedPorts: []string{"8333/tcp", "18333/tcp"}, + Mounts: testcontainers.Mounts(testcontainers.VolumeMount(s.volume, "/data")), + Files: []testcontainers.ContainerFile{ + {Reader: bytes.NewReader(s.identities), ContainerFilePath: "/run/secrets/s3_identities", FileMode: 0444}, + {Reader: bytes.NewReader(s.options), ContainerFilePath: "/etc/seaweedfs/lab.options", FileMode: 0444}, + }, + HostConfigModifier: func(h *container.HostConfig) { + h.Resources.Memory = 512 * 1024 * 1024 + h.Resources.NanoCPUs = 1_000_000_000 + h.Resources.PidsLimit = &pids + h.PortBindings = nat.PortMap{"8333/tcp": {{HostIP: "127.0.0.1"}}, "18333/tcp": {{HostIP: "127.0.0.1"}}} + }, + // The first reachable S3 response must already reject anonymous access. + WaitingFor: wait.ForHTTP("/").WithPort("8333/tcp").WithStatusCodeMatcher(func(code int) bool { firstStatus = code; return true }).WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + s.container = c + if err != nil { + return fmt.Errorf("start S3: %w", err) + } + if firstStatus != http.StatusForbidden { + return fmt.Errorf("first anonymous S3 response was %d, expected 403", firstStatus) + } + s.Endpoint, err = c.PortEndpoint(ctx, "8333/tcp", "http") + if err != nil { + return err + } + s.GRPCAddress, err = c.PortEndpoint(ctx, "18333/tcp", "") + if err != nil { + return err + } + s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey) + if err != nil { + return err + } + s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey) + if err != nil { + return err + } + s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey) + return err +} + +func (s *S3) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return s.container.Stop(ctx, &timeout) +} + +func (s *S3) Start(ctx context.Context) error { + if err := s.container.Start(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Recreate replaces the container while keeping only its named /data volume. +// Bootstrap is deliberately not called: bucket metadata must survive as well. +func (s *S3) Recreate(ctx context.Context) error { + if err := testcontainers.TerminateContainer(s.container); err != nil { + return err + } + s.container = nil + if err := s.launch(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Diagnostics retains bounded startup evidence without exposing fixture keys. +func (s *S3) Diagnostics(ctx context.Context) string { + if s.container == nil { + return "no container" + } + r, err := s.container.Logs(ctx) + if err != nil { + return err.Error() + } + defer r.Close() + redact := strings.NewReplacer(s.AccessKey, "", s.SecretKey, "", s.APIAccessKey, "", s.APISecretKey, "", s.AdminAccessKey, "", s.AdminSecretKey, "") + var lines []string + scan := bufio.NewScanner(io.LimitReader(r, 256*1024)) + for scan.Scan() { + line := scan.Text() + lower := strings.ToLower(line) + if strings.Contains(lower, "volume") || strings.Contains(lower, "master") || strings.Contains(lower, "leader") || strings.Contains(lower, "error") || strings.Contains(lower, "lookup") { + lines = append(lines, redact.Replace(line)) + if len(lines) > 60 { + lines = lines[1:] + } + } + } + return strings.Join(lines, "\n") +} + +func (s *S3) Close() { + if s.container != nil { + _ = testcontainers.TerminateContainer(s.container) + } + if strings.HasPrefix(s.volume, "lab4-s3-test-") { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) }) + } +} diff --git a/migrations/00003_attachments.sql b/migrations/00003_attachments.sql new file mode 100644 index 00000000..abb4756e --- /dev/null +++ b/migrations/00003_attachments.sql @@ -0,0 +1,35 @@ +-- +goose Up +CREATE TABLE files ( + hash bytea PRIMARY KEY CHECK (octet_length(hash) = 32), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + mime_type text NOT NULL CHECK (mime_type IN ('image/png', 'image/jpeg')), + status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + file_hash bytea NOT NULL REFERENCES files(hash), + created_at timestamptz NOT NULL +); + +-- The reconciler must distinguish a shared file from an unused file. +CREATE INDEX attachments_file_hash_idx ON attachments(file_hash); + +CREATE TABLE message_attachments ( + chat_id uuid NOT NULL, + message_id uuid NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments(id), + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY(chat_id, message_id, attachment_id), + UNIQUE(chat_id, message_id, ordinal), + FOREIGN KEY(chat_id, message_id) REFERENCES messages(chat_id, id) +); + +-- +goose Down +DROP TABLE message_attachments; +DROP TABLE attachments; +DROP TABLE files; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh index ae7df48b..081d1023 100755 --- a/scripts/init-secrets.sh +++ b/scripts/init-secrets.sh @@ -14,11 +14,7 @@ for f in $files; do present=$((present + 1)) fi done -if [ "$present" -eq 4 ]; then - echo "secrets already exist in ./secrets, kept" - exit 0 -fi -if [ "$present" -ne 0 ]; then +if [ "$present" -ne 0 ] && [ "$present" -ne 4 ]; then echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2 exit 1 fi @@ -27,16 +23,74 @@ fi random() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n' } -pg=$(random) -kv=$(random) - # The directory is private; the files stay readable for the unprivileged # users inside the containers, which see them through bind mounts. umask 022 mkdir -p secrets chmod 700 secrets -printf '%s\n' "$pg" > secrets/postgres_password -printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url -printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf -printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url -echo "created ./secrets" +if [ "$present" -eq 0 ]; then + pg=$(random) + kv=$(random) + printf '%s\n' "$pg" > secrets/postgres_password + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url + printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf + printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url +fi + +# Read just the bucket setting, never execute the local .env file as a script. +bucket=${S3_BUCKET:-} +if [ -z "$bucket" ] && [ -f .env ]; then + bucket=$(sed -n 's/^S3_BUCKET=//p' .env | tail -n 1 | tr -d '\r') +fi +bucket=${bucket:-messenger-attachments} +case "$bucket" in + *[!a-z0-9.-]*|.*|*.|-*|*-|*..*|*.-*|*-.*) + echo "S3_BUCKET must be a lowercase S3 bucket name" >&2; exit 1 ;; +esac +if [ "${#bucket}" -lt 3 ] || [ "${#bucket}" -gt 63 ]; then + echo "S3_BUCKET must contain 3 to 63 characters" >&2; exit 1 +fi + +s3files="s3_admin_access_key_id s3_admin_secret_access_key s3_access_key_id s3_secret_access_key s3_reconcile_access_key_id s3_reconcile_secret_access_key s3_identities.json" +s3present=0 +for f in $s3files; do + if [ -s "secrets/$f" ]; then s3present=$((s3present + 1)); fi +done +if [ "$s3present" -eq 7 ]; then + if ! grep -Fq "\"Read:$bucket\"" secrets/s3_identities.json; then + echo "existing S3 identities use another bucket; keep S3_BUCKET consistent with the retained volume" >&2 + exit 1 + fi + echo "existing PostgreSQL, Valkey and S3 secrets kept" + exit 0 +fi +if [ "$s3present" -ne 0 ]; then + echo "S3 secrets are incomplete; restore the s3_* secret files together; PostgreSQL and Valkey secrets were kept" >&2 + exit 1 +fi + +admin_key=$(random) +admin_secret=$(random) +api_key=$(random) +api_secret=$(random) +recon_key=$(random) +recon_secret=$(random) +printf '%s\n' "$admin_key" > secrets/s3_admin_access_key_id +printf '%s\n' "$admin_secret" > secrets/s3_admin_secret_access_key +printf '%s\n' "$api_key" > secrets/s3_access_key_id +printf '%s\n' "$api_secret" > secrets/s3_secret_access_key +printf '%s\n' "$recon_key" > secrets/s3_reconcile_access_key_id +printf '%s\n' "$recon_secret" > secrets/s3_reconcile_secret_access_key + +# The server loads all identities before listening. There is no anonymous +# identity, and no post-start allow-all provisioning phase. +cat > secrets/s3_identities.json </dev/null 2>&1; then + stable=$((stable + 1)) + if [ "$stable" -ge 3 ]; then + echo "balancer readiness confirmed" + exit 0 + fi + else + stable=0 + fi + attempt=$((attempt + 1)) + sleep 1 +done +echo "balancer did not become ready within 60 probes" >&2 +exit 1 -- 2.54.0 From 29f45ca0eb0ca060f77a8ada1e20aeca4d7e0785 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:37:21 +0300 Subject: [PATCH 3/6] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D1=8C=D1=82?= =?UTF-8?q?=D0=B5=20=D0=B2=D0=BE=D1=81=D0=BF=D1=80=D0=BE=D0=B8=D0=B7=D0=B2?= =?UTF-8?q?=D0=BE=D0=B4=D0=B8=D0=BC=D1=8B=D0=B5=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D1=8B=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D0=BF=D0=B0=D0=BC=D1=8F=D1=82=D0=B8=20=D0=B8=20=D1=81=D0=BE?= =?UTF-8?q?=D0=B3=D0=BB=D0=B0=D1=81=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 14 + scripts/lab4/decoder-memory.py | 209 ++++++++++++ scripts/lab4/failover.py | 95 ++++++ scripts/lab4/generate-decoder-fixtures.py | 45 +++ scripts/lab4/test_decoder_memory.py | 55 +++ scripts/lab4/test_upload_memory.py | 139 ++++++++ scripts/lab4/upload-memory.py | 376 +++++++++++++++++++++ tools/reconcilebench/main.go | 394 ++++++++++++++++++++++ tools/reconcilebench/main_test.go | 46 +++ tools/reconcilebench/options.go | 43 +++ 10 files changed, 1416 insertions(+) create mode 100644 scripts/lab4/decoder-memory.py create mode 100644 scripts/lab4/failover.py create mode 100644 scripts/lab4/generate-decoder-fixtures.py create mode 100644 scripts/lab4/test_decoder_memory.py create mode 100644 scripts/lab4/test_upload_memory.py create mode 100644 scripts/lab4/upload-memory.py create mode 100644 tools/reconcilebench/main.go create mode 100644 tools/reconcilebench/main_test.go create mode 100644 tools/reconcilebench/options.go diff --git a/Makefile b/Makefile index 21b7346d..96ccf732 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ secrets: up: docker compose up -d --build --wait + sh scripts/wait-ready.sh # Stops and removes the containers; volumes and therefore data are kept. down: @@ -63,6 +64,19 @@ destroy: logs: docker compose logs +# Safe default: inspection only; APPLY=1 explicitly enables repairs. +APPLY ?= 0 +DEEP ?= 0 +.PHONY: reconcile upload-memory reconcile-bench +reconcile: + docker compose --profile recon run --rm reconcile $(if $(filter 1,$(APPLY)),--apply,) $(if $(filter 1,$(DEEP)),--deep,) + +upload-memory: + $(PYTHON) -X utf8 scripts/lab4/upload-memory.py --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/upload-memory.json" + +reconcile-bench: + go run ./tools/reconcilebench --output "$(EVIDENCE_DIR)/track2/results.json" + # Which container answers under which X-Instance-Id. instances: sh scripts/instances.sh diff --git a/scripts/lab4/decoder-memory.py b/scripts/lab4/decoder-memory.py new file mode 100644 index 00000000..230db079 --- /dev/null +++ b/scripts/lab4/decoder-memory.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits. + +Run with python -X utf8 -B. Default is one upload; no container lifecycle actions. +""" +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import struct +import subprocess +import sys +import threading +import time +from urllib.parse import urlsplit +import zlib + +sys.dont_write_bytecode = True +_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) +memory = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(memory) + + +def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536): + """Stream zero scanlines, including filter bytes; never allocate a full row.""" + if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000: + raise ValueError('invalid stress dimensions or chunk size') + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + encoder = zlib.compressobj(1) + remaining = height * (width * 8 + 1) + zeros = bytes(min(chunk_bytes, remaining)) + with path.open('wb') as out: + out.write(b'\x89PNG\r\n\x1a\n') + out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0))) + while remaining: + count = min(len(zeros), remaining) + compressed = encoder.compress(zeros[:count]) + if compressed: + out.write(memory.png_chunk(b'IDAT', compressed)) + remaining -= count + out.write(memory.png_chunk(b'IDAT', encoder.flush())) + out.write(memory.png_chunk(b'IEND', b'')) + + +def fixture_info(content, mime): + if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33: + width, height, depth, color = struct.unpack_from('>IIBB', content, 16) + return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color} + if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'): + pos = 2 + while pos + 4 <= len(content): + if content[pos] != 255: + break + while pos < len(content) and content[pos] == 255: + pos += 1 + marker = content[pos] + pos += 1 + length = struct.unpack_from('>H', content, pos)[0] + frame = content[pos + 2:pos + length] + pos += length + if marker in (0xc0, 0xc1, 0xc2): + height, width = struct.unpack_from('>HH', frame, 1) + count = frame[5] + return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count, + 'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]} + if length < 2 or marker == 0xda: + break + raise ValueError('fixture MIME/header mismatch or missing image header') + + +def read_fixture(path, mime): + if not 0 < path.stat().st_size <= 10 * 1024 * 1024: + raise ValueError('fixture must be between 1 byte and 10 MiB') + content = path.read_bytes() + info = fixture_info(content, mime) + if not 0 < info['width'] * info['height'] <= 40_000_000: + raise ValueError('fixture exceeds 40M pixels') + return content, info + + +def api_limits(ids): + template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}' + result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())} + + +def experiment(args, content, report): + sys.path.insert(0, str(memory.ROOT / 'tests')) + import client as module + ids = memory.api_containers(args) + report['api_limits'] = api_limits(ids) + if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids): + raise RuntimeError('stress uploads require every API container limit to be >=2 GiB') + report['state_before'] = memory.container_states(ids) + bootstrap = module.Client(args.base_url, timeout=args.timeout) + actor, reader, _, chat = bootstrap.bootstrap() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + with memory.DockerStats(ids) as stats: + stats.mark(args.label, 'baseline') + stats.wait_for_baseline(args.label) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'], + barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)] + stats.mark(args.label, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + report['duration_s'] = time.monotonic() - started + time.sleep(args.tail_seconds) + stats.mark(args.label, 'idle') + report['memory_samples'] = stats.snapshot(args.label) + report['sampling_error'] = stats.error + report['state_after'] = memory.container_states(ids) + stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed'] + and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids) + report['restart_or_oom_detected'] = not stable + try: + report['memory'] = memory.summarize_memory(report['memory_samples'], ids) + except RuntimeError as exc: + report['memory_error'] = str(exc) + report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results]) + accepted = [r for r in results if r['ok']] + report['accepted'] = len(accepted) + report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)) + report['latency'] = memory.latency_summary([r['latency_ms'] for r in results]) + report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + report['results'] = results + report['download'] = {'ok': False} + if accepted: + try: + response = bootstrap.call('GET', '/api/v1/attachments/' + accepted[0]['attachment_id'] + '/content', actor=reader) + body = response['bytes'] + report['download'] = {'ok': body == content and response['headers'].get_content_type() == args.mime, + 'status': response['status'], 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + except Exception as exc: + report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__ + report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable + and not report.get('memory_error') and not report['sampling_error'] + and report['observed_max_inflight'] == args.concurrency) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--file', type=Path) + parser.add_argument('--mime', choices=('image/png', 'image/jpeg')) + parser.add_argument('--label', default='decoder') + parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml') + parser.add_argument('--project-name') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--output', type=Path) + parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker') + parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input') + args = parser.parse_args() + if args.generate_wide: + write_rgba64_png(args.generate_wide) + content, info = read_fixture(args.generate_wide, 'image/png') + print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})) + return 0 + if not args.file or not args.mime: + parser.error('--file and --mime are required') + if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s') + if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label): + parser.error('label must be 1-40 safe ASCII characters') + url = urlsplit(args.base_url) + if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment: + parser.error('base URL must be HTTP(S) without credentials/query/fragment') + content, info = read_fixture(args.file, args.mime) + report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info, + 'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(), + 'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'measurement': {'source': 'docker stats JSON stream, formatted working-set values', + 'limitation': 'sampled/rounded peaks can miss instantaneous maxima', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}} + if args.check_file: + print(json.dumps(report, indent=2)) + return 0 + if not args.output: + parser.error('--output is required for a live run') + try: + experiment(args, content, report) + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure': + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/lab4/failover.py b/scripts/lab4/failover.py new file mode 100644 index 00000000..a16b81f2 --- /dev/null +++ b/scripts/lab4/failover.py @@ -0,0 +1,95 @@ +"""Stop each actual uploader replica and verify its bytes through the survivor.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def instances(): + text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT, + encoding='utf-8') + result = [] + for line in text.splitlines(): + container, replica, state, alias, image = line.split() + result.append({'container': container, 'replica': replica, + 'state': state, 'alias': alias, 'image': image}) + return result + + +def restore(c): + subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT, + check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id'] + for _ in range(6)} + if len(observed) >= 2: + return + time.sleep(0.3) + raise AssertionError('two replicas did not return after restore') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json') + args = p.parse_args() + c = Client(args.base_url, timeout=10) + actor, reader, _, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + wanted = instances() + check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted), + 'this experiment requires two running API replicas') + report = {'scenario': 'stop actual upload-serving replica; download via survivor', + 'cases': [], 'ok': False} + try: + for target in wanted: + upload = None + for _ in range(10): + attempt = c.call('POST', '/api/v1/chats/' + chat['id'] + + '/attachments?filename=sample.png', 201, + actor=actor, raw=data, + headers={'Content-Type': 'image/png'}, schema='Attachment') + if attempt['headers']['X-Instance-Id'] == target['alias']: + upload = attempt + break + check(upload is not None, 'target replica did not serve an upload') + before = instances() + env = dict(os.environ, TARGET_INSTANCE_ID=target['alias']) + started = time.monotonic() + subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env, + check=True, stdout=subprocess.DEVNULL, timeout=15) + stop_seconds = time.monotonic() - started + downloaded = c.call('GET', '/api/v1/attachments/' + + upload['json']['id'] + '/content', actor=reader) + survivor = downloaded['headers']['X-Instance-Id'] + check(survivor != target['alias'], 'stopped uploader still served download') + check(downloaded['bytes'] == data, 'bytes changed after uploader stop') + after = instances() + check(any(x['container'] == target['container'] and x['state'] != 'running' + for x in after), 'uploader was not left stopped') + report['cases'].append({'uploader': target, 'download_instance': survivor, + 'stop_seconds': round(stop_seconds, 3), + 'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(), + 'same_bytes': True, 'before': before, 'after': after}) + restore(c) + report['ok'] = True + finally: + # This script only restores the API containers that it deliberately stopped. + restore(c) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']), + 'output': str(args.output)})) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/generate-decoder-fixtures.py b/scripts/lab4/generate-decoder-fixtures.py new file mode 100644 index 00000000..dfc752ba --- /dev/null +++ b/scripts/lab4/generate-decoder-fixtures.py @@ -0,0 +1,45 @@ +"""Generate optional 40 MP regression images; requires Pillow==11.1.0.""" +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import sys + +sys.dont_write_bytecode = True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output-dir', type=Path, default=Path('bin/decoder-fixtures')) + args = parser.parse_args() + try: + from PIL import Image + except ImportError: + parser.error('install optional fixture dependency: pip install Pillow==11.1.0') + spec = importlib.util.spec_from_file_location('decoder_memory', + Path(__file__).with_name('decoder-memory.py')) + decoder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(decoder) + args.output_dir.mkdir(parents=True, exist_ok=True) + manifest = {'pillow_version': Image.__version__, 'fixtures': []} + for mode, color in [('RGB', (25, 90, 160)), ('CMYK', (20, 40, 60, 10))]: + path = args.output_dir / ('progressive-' + mode.lower() + '-444-40mp.jpg') + with Image.new(mode, (8000, 5000), color) as image: + image.save(path, format='JPEG', progressive=True, subsampling=0, quality=85) + content, info = decoder.read_fixture(path, 'image/jpeg') + if not info['progressive'] or any(value != 0x11 for value in info['sampling']): + raise ValueError('encoder did not produce progressive 4:4:4') + manifest['fixtures'].append({'name': path.name, 'mime': 'image/jpeg', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + wide = args.output_dir / 'wide-rgba64-40mp.png' + decoder.write_rgba64_png(wide) + content, info = decoder.read_fixture(wide, 'image/png') + manifest['fixtures'].append({'name': wide.name, 'mime': 'image/png', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + (args.output_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8') + print(json.dumps(manifest, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/test_decoder_memory.py b/scripts/lab4/test_decoder_memory.py new file mode 100644 index 00000000..45e198ef --- /dev/null +++ b/scripts/lab4/test_decoder_memory.py @@ -0,0 +1,55 @@ +"""Offline checks for bounded stress fixtures and format identification.""" +import importlib.util +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +class DecoderMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = Path(__file__).with_name('decoder-memory.py') + cls.module = None + if path.exists(): + spec = importlib.util.spec_from_file_location('decoder_memory', path) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def setUp(self): + self.assertIsNotNone(self.module, 'decoder-memory.py is not implemented') + + def test_streamed_rgba64_png_has_valid_crc_and_zero_pixels(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'wide.png' + self.module.write_rgba64_png(path, width=19, height=3, chunk_bytes=7) + payload = path.read_bytes() + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, body = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + body) & 0xffffffff) + if kind == b'IDAT': + compressed.extend(body) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(3 * (19 * 8 + 1))) + self.assertEqual(self.module.fixture_info(payload, 'image/png'), + {'width': 19, 'height': 3, 'bit_depth': 16, 'color_type': 6}) + + def test_progressive_jpeg_444_components_are_identified(self): + sof = b'\x08' + struct.pack('>HHB', 5000, 8000, 3) + bytes([1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]) + payload = b'\xff\xd8\xff\xc2' + struct.pack('>H', len(sof) + 2) + sof + info = self.module.fixture_info(payload, 'image/jpeg') + self.assertEqual((info['width'], info['height'], info['components']), (8000, 5000, 3)) + self.assertTrue(info['progressive']) + self.assertEqual(info['sampling'], [0x11, 0x11, 0x11]) + + def test_fixture_byte_cap_is_checked_before_reading(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'oversized.png' + with path.open('wb') as out: + out.truncate(10 * 1024 * 1024 + 1) + with self.assertRaises(ValueError): + self.module.read_fixture(path, 'image/png') diff --git a/scripts/lab4/test_upload_memory.py b/scripts/lab4/test_upload_memory.py new file mode 100644 index 00000000..63c75620 --- /dev/null +++ b/scripts/lab4/test_upload_memory.py @@ -0,0 +1,139 @@ +"""Offline checks for reproducible PNGs and Docker memory evidence.""" +import importlib.util +import json +from pathlib import Path +import struct +import unittest +from types import SimpleNamespace +import io +import zlib + + +class UploadMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def inspect_png(self, payload, width, height): + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, chunks, compressed = 8, [], bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind = payload[pos + 4:pos + 8] + data = payload[pos + 8:pos + 8 + size] + crc = struct.unpack_from('>I', payload, pos + 8 + size)[0] + self.assertEqual(crc, zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (width, height, 8, 2, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + chunks.append(kind) + pos += size + 12 + self.assertEqual(pos, len(payload)) + self.assertEqual(chunks[0], b'IHDR') + self.assertEqual(chunks[-1], b'IEND') + self.assertIn(b'npAD', chunks) + rows = zlib.decompress(compressed) + self.assertEqual(len(rows), height * (width * 3 + 1)) + self.assertTrue(all(rows[row * (width * 3 + 1)] == 0 for row in range(height))) + + def test_seeded_png_has_correct_crc_pixels_and_exact_size(self): + first = self.module.make_png(32, 24, 4096, 123) + self.assertEqual(len(first), 4096) + self.assertEqual(first, self.module.make_png(32, 24, 4096, 123)) + self.assertNotEqual(first, self.module.make_png(32, 24, 4096, 124)) + self.inspect_png(first, 32, 24) + + def test_both_experiment_payloads_decode_and_reach_exact_limits(self): + for name, width, height, size, seed in self.module.CASES: + with self.subTest(name=name): + payload = self.module.make_png(width, height, size, seed) + self.assertEqual(len(payload), size) + self.assertLessEqual(len(payload), 10 * 1024 * 1024) + self.assertLessEqual(width * height, 40_000_000) + self.inspect_png(payload, width, height) + + def test_impossible_png_size_is_rejected(self): + with self.assertRaises(ValueError): + self.module.make_png(32, 32, 64, 1) + + def test_docker_memory_uses_binary_and_decimal_units(self): + for raw, expected in [('1.5MiB / 512MiB', 1572864), ('2MB / 512MB', 2000000), ('4096B / 1GiB', 4096)]: + self.assertEqual(self.module.memory_bytes(raw), expected) + with self.assertRaises(ValueError): + self.module.memory_bytes('not available') + + def test_memory_summary_separates_baseline_and_load_for_each_replica(self): + samples = [ + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 10}, + {'container_id': 'b', 'phase': 'baseline', 'used_bytes': 20}, + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 12}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 35}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 30}, + {'container_id': 'b', 'phase': 'load', 'used_bytes': 50}, + ] + result = self.module.summarize_memory(samples, ['a', 'b']) + self.assertEqual(result['a']['baseline_bytes'], 12) + self.assertEqual(result['a']['sampled_peak_bytes'], 35) + self.assertEqual(result['a']['delta_bytes'], 23) + self.assertEqual(result['b']['sampled_peak_bytes'], 50) + with self.assertRaises(RuntimeError): + self.module.summarize_memory(samples, ['a', 'b', 'missing']) + + def test_latency_percentiles_and_overlap_are_explicit(self): + self.assertEqual(self.module.latency_summary(list(range(1, 11))), + {'p50_ms': 5.5, 'p95_ms': 10, 'max_ms': 10}) + self.assertEqual(self.module.max_inflight([(0, 4), (1, 3), (2, 5)]), 3) + self.assertEqual(self.module.max_inflight([(0, 1), (1, 2)]), 1) + + def test_docker_stream_ignores_ansi_only_lines_between_samples(self): + cid = 'a' * 64 + row = json.dumps({'ID': cid, 'Name': 'lab4-api-1', 'MemUsage': '12MiB / 512MiB'}) + stats = self.module.DockerStats([cid]) + stats.process = SimpleNamespace(stdout=iter(['\x1b[H' + row + '\n', ' \x1b[K\n', row + '\n'])) + stats.mark('small', 'baseline') + stats._read() + self.assertIsNone(stats.error) + self.assertEqual(len(stats.samples), 2) + + def test_rgba64_profile_has_valid_header_crc_and_zero_scanlines(self): + payload = self.module.make_rgba64_png(8, 5) + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, data = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (8, 5, 16, 6, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(5 * (1 + 8 * 8))) + self.assertEqual(pos, len(payload)) + + def test_max_pixels_profile_fits_byte_limit_without_full_raw_buffer(self): + payload = self.module.make_rgba64_png(8000, 5000) + self.assertLess(len(payload), 10 * 1024 * 1024) + self.assertEqual(struct.unpack_from('>IIBB', payload, 16), (8000, 5000, 16, 6)) + + def test_upload_uses_requested_jpeg_mime(self): + captured = [] + class Opener: + def open(self, request, timeout): + captured.append(request) + response = io.BytesIO(b'') + response.code, response.headers = 415, {} + return response + client = SimpleNamespace(base='http://localhost', opener=Opener(), timeout=1) + import threading + result = self.module.upload(None, client, {'token': 'synthetic'}, {'id': 'chat'}, b'jpeg', 'hash', + threading.Barrier(1), 0, 'decoder', mime='image/jpeg') + self.assertEqual(captured[0].get_header('Content-type'), 'image/jpeg') + self.assertTrue(captured[0].full_url.endswith('decoder-0.jpg')) + self.assertEqual(result['status'], 415) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/lab4/upload-memory.py b/scripts/lab4/upload-memory.py new file mode 100644 index 00000000..81515831 --- /dev/null +++ b/scripts/lab4/upload-memory.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Concurrent upload experiment; stdlib only, no stand lifecycle operations.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import re +import statistics +import struct +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from urllib.parse import urlsplit +import zlib + + +ROOT = Path(__file__).resolve().parents[2] +sys.dont_write_bytecode = True + +CASES = ( + ('small', 512, 512, 1024 * 1024, 41), + ('limit', 1600, 1600, 10 * 1024 * 1024, 42), +) +MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None) + + +def png_chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def make_png(width, height, size, seed): + rng = random.Random(seed) + rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1)) + ending = png_chunk(b'IEND', b'') + gap = size - len(prefix) - len(ending) + if gap < 0 or 0 < gap < 12: + raise ValueError('target cannot contain this PNG') + # Ancillary, private, reserved bit valid: unknown npAD is safely ignored. + padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b'' + return prefix + padding + ending + + +def make_rgba64_png(width, height): + encoder = zlib.compressobj(1) + row = bytes(1 + width * 8) + pieces = [encoder.compress(row) for _ in range(height)] + pieces.append(encoder.flush()) + header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + + png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b'')) + + +def payload_for(case): + name, width, height, size, seed = case + return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed) + + +def memory_bytes(value): + match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip()) + if not match: + raise ValueError('unrecognized docker memory value') + units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2, + 'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4} + return round(float(match[1]) * units[match[2]]) + + +def summarize_memory(samples, container_ids): + result = {} + for container_id in container_ids: + baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline'] + load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load'] + if not baseline or not load: + raise RuntimeError('missing baseline or load sample for an API replica') + before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load) + result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before, + 'baseline_samples': len(baseline), 'load_samples': len(load)} + return result + + +def latency_summary(values): + ordered = sorted(values) + return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1], + 'max_ms': ordered[-1]} + + +def max_inflight(intervals): + active, peak = 0, 0 + events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals] + for _, delta in sorted(events): + active += delta + peak = max(peak, active) + return peak + + +class DockerStats: + def __init__(self, container_ids): + self.ids = container_ids + self.samples = [] + self.lock = threading.Condition() + self.case, self.phase = '', 'idle' + self.error = None + self.started = time.monotonic() + + def __enter__(self): + self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, encoding='utf-8', errors='replace', bufsize=1) + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + return self + + def _read(self): + for line in self.process.stdout: + try: + cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + if not cleaned: + continue + raw = json.loads(cleaned) + reported_id = raw.get('ID') or raw.get('Container', '') + container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid)) + used = memory_bytes(raw['MemUsage']) + limit = memory_bytes(raw['MemUsage'].split('/')[1]) + if used == 0: + continue + with self.lock: + self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used, + 'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6), + 'case': self.case, 'phase': self.phase}) + self.lock.notify_all() + except (ValueError, KeyError, IndexError, StopIteration) as exc: + with self.lock: + self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120]) + self.lock.notify_all() + with self.lock: + self.lock.notify_all() + + def mark(self, case, phase): + with self.lock: + self.case, self.phase = case, phase + + def snapshot(self, case): + with self.lock: + return [dict(s) for s in self.samples if s['case'] == case] + + def wait_for_baseline(self, case, timeout=20): + deadline = time.monotonic() + timeout + with self.lock: + while True: + seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'} + if seen == set(self.ids): + return + if self.error or self.process.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError('docker stats baseline unavailable') + self.lock.wait(min(.1, max(0, deadline - time.monotonic()))) + + def __exit__(self, *_): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + self.thread.join(timeout=5) + self.process.stdout.close() + + +def api_containers(args): + command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)] + if args.project_name: + command.extend(['--project-name', args.project_name]) + result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + ids = result.stdout.split() + if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids): + raise RuntimeError('compose has no running API containers') + return ids + + +def container_states(container_ids): + template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},' + '"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},' + '"exit_code":{{json .State.ExitCode}}}') + result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())} + + +def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'): + result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)} + barrier.wait(timeout=30) + started = time.monotonic() + try: + suffix = '.jpg' if mime == 'image/jpeg' else '.png' + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix + req = urllib.request.Request(client.base + path, data=content, method='POST', headers={ + 'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime, + 'Content-Length': str(len(content)), 'Accept': 'application/json'}) + try: + response = client.opener.open(req, timeout=client.timeout) + except urllib.error.HTTPError as exc: + response = exc + with response: + result['status'] = response.code + result['instance_id'] = response.headers.get('X-Instance-Id') + body = response.read(65537) + if result['status'] != 201: + result['error'] = 'http_error' + elif len(body) > 65536: + result['error'] = 'oversized_metadata' + else: + attachment = json.loads(body) + client_module.shape(attachment, 'Attachment') + valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content) + and attachment['status'] == 'ready' and attachment['owner_id'] == actor['id'] + and attachment['chat_id'] == chat['id']) + result['ok'] = valid + result['attachment_id'] = attachment['id'] + if not valid: + result['error'] = 'attachment_metadata_mismatch' + except Exception as exc: + # Exception text and response bodies are deliberately excluded from evidence. + result['error'] = type(exc).__name__ + finished = time.monotonic() + result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3)) + return result + + +def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case): + name, width, height, size, seed = case + content = payload_for(case) + size = len(content) + checksum = hashlib.sha256(content).hexdigest() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + state_before = container_states(stats.ids) + stats.mark(name, 'baseline') + stats.wait_for_baseline(name) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name) + for index, client in enumerate(clients)] + stats.mark(name, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + duration = time.monotonic() - started + # Retain a tail because docker stats reports its own sampled windows. + time.sleep(args.tail_seconds) + stats.mark(name, 'idle') + samples = stats.snapshot(name) + memory_error = None + try: + memory = summarize_memory(samples, stats.ids) + except RuntimeError as exc: + memory, memory_error = {}, str(exc) + intervals = [(r['_started'], r['_finished']) for r in results] + peak_requests = max_inflight(intervals) + successes = [r for r in results if r['ok']] + download = {'ok': False} + if successes: + try: + response = bootstrap_client.call('GET', '/api/v1/attachments/' + successes[0]['attachment_id'] + '/content', actor=reader) + downloaded = response['bytes'] + download = {'ok': downloaded == content and response['headers'].get_content_type() == 'image/png', + 'bytes': len(downloaded), 'sha256': hashlib.sha256(downloaded).hexdigest(), + 'status': response['status']} + except Exception as exc: + download['error'] = type(exc).__name__ + distinct = len({r['attachment_id'] for r in successes}) + state_after = container_states(stats.ids) + stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed'] + and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum, + 'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2, + 'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4), + 'requests': args.concurrency, 'attempted_bytes': size * args.concurrency, + 'successful_bytes': size * len(successes), 'accepted': len(successes), + 'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)), + 'errors': len(results) - len(successes), 'duration_s': round(duration, 6), + 'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct, + 'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory, + 'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download, + 'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable, + 'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10 + and download['ok'] and not memory_error and stable)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml') + parser.add_argument('--project-name', default=None) + parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here') + parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard', + help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)') + parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker') + args = parser.parse_args() + selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,) + if args.check_generator: + print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload), + 'sha256': hashlib.sha256(payload).hexdigest()} + for name, width, height, size, seed in selected_cases + for payload in [payload_for((name, width, height, size, seed))]], indent=2)) + return 0 + if not args.output: + parser.error('--output is required for live execution') + if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s') + target = urlsplit(args.base_url) + if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment: + parser.error('--base-url must be HTTP(S) without credentials, query or fragment') + report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False, + 'driver': {'os': platform.system(), 'architecture': platform.machine(), + 'logical_cpus': os.cpu_count(), 'python': platform.python_version()}, + 'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second', + 'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail', + 'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds, + 'same_bytes_per_case': True, 'p95_method': 'nearest rank', + 'profile': args.profile, + 'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard' + else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []} + try: + sys.path.insert(0, str(ROOT / 'tests')) + import client as client_module + ids = api_containers(args) + report['api_container_ids'] = ids + bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout) + report['stage'] = 'bootstrap' + actor, reader, _, chat = bootstrap_client.bootstrap() + with DockerStats(ids) as stats: + for case in selected_cases: + report['stage'] = case[0] + report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case)) + if stats.error: + report['sampling_error'] = stats.error + report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error + report['stage'] = 'complete' + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)): + # Public helper diagnostics contain only paths/schema names, never values or credentials. + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'output': str(args.output), + 'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'], + 'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'], + 'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']], + 'error': report.get('error')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go new file mode 100644 index 00000000..b1b3ab47 --- /dev/null +++ b/tools/reconcilebench/main.go @@ -0,0 +1,394 @@ +// Command reconcilebench measures reconciliation against disposable real +// PostgreSQL and S3 services, independently of the compose stand. +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func main() { + o, err := parseOptions(os.Args[1:], os.Stderr) + if errors.Is(err, flag.ErrHelp) { + return + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + var records bytes.Buffer + var resultOut io.Writer = os.Stdout + if o.output != "" { + resultOut = io.MultiWriter(os.Stdout, &records) + } + err = exercise(ctx, o, resultOut, os.Stderr) + if o.output != "" { + if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil { + err = errors.Join(err, fmt.Errorf("save results: %w", saveErr)) + } + } + cancel() + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "reconcilebench:", err) + os.Exit(1) + } +} + +func writeResults(path string, records []byte) error { + decoder := json.NewDecoder(bytes.NewReader(records)) + rows := make([]json.RawMessage, 0) + for { + var row json.RawMessage + err := decoder.Decode(&row) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + rows = append(rows, row) + } + raw, err := json.MarshalIndent(rows, "", " ") + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0644) +} + +type measurement struct { + Phase string `json:"phase"` + Mode string `json:"mode"` + TargetFiles int `json:"target_files"` + GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"` + Stats attachment.Stats `json:"stats"` + Findings []attachment.Finding `json:"findings,omitempty"` +} + +// Heap samples cover this Go process during Run. They exclude service processes +// and are deliberately not described as process RSS or total deployment memory. +func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) { + done := make(chan struct{}) + peakCh := make(chan uint64, 1) + go func() { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + peak := mem.HeapAlloc + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + case <-done: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + peakCh <- peak + return + } + } + }() + m := measurement{Phase: phase, Mode: mode, TargetFiles: n} + stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) + close(done) + m.GoHeapPeakBytes = <-peakCh + m.Stats = stats + if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil { + return stats, writeErr + } + return stats, err +} + +func exercise(ctx context.Context, o options, out, progress io.Writer) error { + db, err := testenv.StartPostgres(ctx) + if err != nil { + return err + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + return err + } + defer s3.Close() + if err = json.NewEncoder(out).Encode(map[string]any{ + "phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image, + "grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(), + "memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3", + }); err != nil { + return err + } + f, err := seedFaults(ctx, db.Pool, s3.Store) + if err != nil { + return err + } + fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait) + if err = waitForAge(ctx, o.wait, progress); err != nil { + return err + } + if err = f.seedFresh(ctx); err != nil { + return err + } + r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace) + if err != nil { + return err + } + for _, run := range []struct { + mode string + apply bool + repairs, findings int64 + }{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} { + stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out) + if err != nil { + return err + } + if stats.Repaired != run.repairs || stats.Findings != run.findings { + return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats) + } + if err = f.assertState(ctx, run.apply); err != nil { + return fmt.Errorf("fault %s: %w", run.mode, err) + } + } + fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved") + // These resources were created above solely for this run; no compose + // endpoint or pre-existing database/bucket is ever accepted by this tool. + if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files`); err != nil { + return err + } + if err = emptyBucket(ctx, s3.Store); err != nil { + return err + } + seeded := 0 + for _, n := range o.sizes { + for i := seeded; i < n; i++ { + if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil { + return err + } + if (i+1)%500 == 0 { + fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1) + } + } + seeded = n + stats, err := measure(ctx, r, "cost", "dry", n, false, false, out) + if err != nil { + return err + } + if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 { + return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats) + } + } + return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes}) +} + +func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error { + start := time.Now() + timer := time.NewTimer(d) + defer timer.Stop() + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + case <-ticker.C: + fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d) + } + } +} + +type benchFile struct { + hash []byte + before, after string +} +type benchObject struct { + key string + after bool +} +type faultFixture struct { + db *pgxpool.Pool + blobs attachment.Blobs + user, chat uuid.UUID + files []benchFile + objects []benchObject +} + +func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) { + f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()} + if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil { + return nil, err + } + if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil { + return nil, err + } + if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil { + return nil, err + } + orphan := sha256.Sum256([]byte("old orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil { + return nil, err + } + for _, c := range []struct { + label, status, after string + used, present, corrupt bool + }{ + {"pending", "pending", "", false, true, false}, + {"unused", "ready", "", false, true, false}, + {"used", "ready", "ready", true, true, false}, + {"corrupt", "ready", "ready", true, true, true}, + {"missing", "ready", "failed", true, false, false}, + {"already_failed", "failed", "failed", true, false, false}, + } { + if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil { + return nil, err + } + } + return f, nil +} + +func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error { + if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + f.objects = append(f.objects, benchObject{key: key, after: keep}) + return nil +} + +func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error { + body := []byte("fixture/" + label) + hash := sha256.Sum256(body) + created := time.Now().Add(-2 * time.Hour) + if fresh { + created = time.Now() + } + if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil { + return err + } + f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after}) + if used { + id, message := uuid.New(), uuid.New() + if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil { + return err + } + } + if present { + if corrupt { + body[0] ^= 1 + } + return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "") + } + return nil +} + +func (f *faultFixture) seedFresh(ctx context.Context) error { + if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil { + return err + } + orphan := sha256.Sum256([]byte("fresh orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil { + return err + } + return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true) +} + +func (f *faultFixture) assertState(ctx context.Context, applied bool) error { + for _, file := range f.files { + var status string + err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + want := file.before + if applied { + want = file.after + } + if status != want { + return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want) + } + } + for _, obj := range f.objects { + _, err := f.blobs.Stat(ctx, obj.key) + want := !applied || obj.after + if err != nil && !errors.Is(err, attachment.ErrObjectMissing) { + return err + } + if (err == nil) != want { + return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want) + } + } + var attachments, links int + if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil { + return err + } + if attachments != 4 || links != 4 { + return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links) + } + return nil +} + +func emptyBucket(ctx context.Context, blobs attachment.Blobs) error { + for { + page, err := blobs.List(ctx, "", "", 500) + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + for _, obj := range page { + if err := blobs.Delete(ctx, obj.Key); err != nil { + return err + } + } + } +} + +func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error { + body := fmt.Sprintf("scale/%09d", i) + hash := sha256.Sum256([]byte(body)) + if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + _, err := db.Exec(ctx, `WITH f AS ( + INSERT INTO files(hash,size_bytes,mime_type,status,created_at) + VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user) + return err +} diff --git a/tools/reconcilebench/main_test.go b/tools/reconcilebench/main_test.go new file mode 100644 index 00000000..e77ecf01 --- /dev/null +++ b/tools/reconcilebench/main_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestOptionsProtectRealAgingAndValidateSizes(t *testing.T) { + got, err := parseOptions(nil, io.Discard) + if err != nil || got.grace != 2*time.Minute || got.wait != 125*time.Second || !reflect.DeepEqual(got.sizes, []int{100, 1000, 5000}) { + t.Fatalf("defaults: %+v %v", got, err) + } + for _, args := range [][]string{{"--grace=1m"}, {"--wait=2m"}, {"--sizes=0"}, {"--sizes=10,10"}, {"--sizes=10,5"}, {"--sizes=oops"}, {"extra"}} { + if _, err := parseOptions(args, io.Discard); err == nil { + t.Errorf("accepted unsafe options %v", args) + } + } + got, err = parseOptions([]string{"--sizes=2,7", "--grace=3m", "--wait=3m2s"}, io.Discard) + if err != nil || !reflect.DeepEqual(got.sizes, []int{2, 7}) { + t.Fatalf("explicit options: %+v %v", got, err) + } +} + +func TestOutputOptionWritesStructuredJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "track2", "results.json") + o, err := parseOptions([]string{"--output", path}, io.Discard) + if err != nil || o.output != path { + t.Fatalf("output flag: %+v %v", o, err) + } + if err = writeResults(path, []byte("{\"phase\":\"fault\"}\n{\"phase\":\"complete\"}\n")); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err = json.Unmarshal(raw, &rows); err != nil || len(rows) != 2 || rows[1]["phase"] != "complete" { + t.Fatalf("results JSON: %s %v", raw, err) + } +} diff --git a/tools/reconcilebench/options.go b/tools/reconcilebench/options.go new file mode 100644 index 00000000..5e6344e8 --- /dev/null +++ b/tools/reconcilebench/options.go @@ -0,0 +1,43 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +type options struct { + grace, wait time.Duration + sizes []int + output string +} + +func parseOptions(args []string, out io.Writer) (options, error) { + var o options + f := flag.NewFlagSet("reconcilebench", flag.ContinueOnError) + f.SetOutput(out) + f.DurationVar(&o.grace, "grace", 2*time.Minute, "reconciler grace (minimum 2m)") + f.DurationVar(&o.wait, "wait", 125*time.Second, "actual S3 aging wait (must exceed grace by at least 1s)") + f.StringVar(&o.output, "output", "", "also save measurement records as a JSON array at this path") + sizes := f.String("sizes", "100,1000,5000", "strictly increasing referenced object counts") + if err := f.Parse(args); err != nil { + return o, err + } + if f.NArg() != 0 || o.grace < 2*time.Minute || o.wait-o.grace < time.Second { + return o, errors.New("no positional arguments; grace >= 2m and wait >= grace+1s required") + } + last := 0 + for _, raw := range strings.Split(*sizes, ",") { + n, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || n <= last { + return o, fmt.Errorf("sizes must be strictly increasing positive integers: %q", raw) + } + o.sizes = append(o.sizes, n) + last = n + } + return o, nil +} -- 2.54.0 From fc3078ed304ae4ecc93745c36e51e2a12a87ea4f Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:40:17 +0300 Subject: [PATCH 4/6] =?UTF-8?q?=D0=9E=D0=BF=D0=B8=D1=88=D0=B8=D1=82=D0=B5?= =?UTF-8?q?=20=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA,=20=D1=80=D0=B5=D0=B7?= =?UTF-8?q?=D1=83=D0=BB=D1=8C=D1=82=D0=B0=D1=82=D1=8B=20=D0=B8=20=D0=BE?= =?UTF-8?q?=D0=B3=D1=80=D0=B0=D0=BD=D0=B8=D1=87=D0=B5=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/pull_request_template.md | 36 ++++--- REPORT.md | 178 +++++++++++++++++++++++++++---- 2 files changed, 182 insertions(+), 32 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..70affc3a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,26 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор указывается при публикации. Группа: БПИ2301. +Целевая оценка: **5**. +Трек 2: **согласованность двух хранилищ**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Перенесены API/БД/сессии/балансировка/идемпотентность lab3 и её серии экспериментов. Добавлены приватный SeaweedFS, ограниченные S3 identities, потоковые PNG/JPEG upload/download и проверка изображений, content-addressed `files` и отдельные ACL-вложения с UUID, атомарные связи сообщений и reconciler с dry-run/apply/deep. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Повторная загрузка одинаковых байтов создаёт новое логическое вложение, сохраняя единственную постоянную копию содержимого. Сборщик сериализуется с публикацией по хэшу, защищает свежие и используемые данные, обнаруживает missing/orphan и безопасно повторяет исправления. + +Запуск: на Windows/Git Bash `export PYTHONUTF8=1`; затем `cp .env.example .env`, `make secrets`, `make up`, `make test PYTHON=python`, `make check PYTHON=python`, `make unit`, `make vet`. Полные команды, решения, границы согласованности, результаты и ограничения измерений — в [REPORT.md](../REPORT.md), исходные данные — в `evidence/`. + +Изменения базы lab3, выделенные при переносе: обычный shutdown увеличен для активных бинарных запросов; старт ожидает свежие ответы LB. После ревью исправлены оценка памяти progressive JPEG/широкого PNG, дедлайн GET метаданных и классификация обрывов S3. S3 bootstrap ждёт регистрации сохранённых volumes, а не только ответа gateway. + +Ограничения: один узел каждого хранилища и LB; HTTP только локально; нет удаления клиентских вложений и резервного восстановления повреждённых байтов. Дедупликация экономит хранение, не трафик; полная проверка изображения требует ограниченной, но существенной памяти. Числа лабораторного опыта не являются production SLA. + +Источники: контракт репозитория, документация SeaweedFS/minio-go/PostgreSQL/S3, ссылки в REPORT. + +ИИ-ассистент использовался как парный программист: генерация кода, тестов и черновиков документации по моим архитектурным решениям с последующим ревью каждого модуля и переработкой по замечаниям; выбор стека, структура проекта, модель данных, обработка ошибок, логирование и границы модулей определены мной; все проверки воспроизведены мной вручную. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..77fe5799 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,179 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Цель: **5, трек 2 — согласованность PostgreSQL и S3**. Автор и ссылка на PR указываются при публикации. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Compose v2, Git Bash и Make. Для Go-тестов — Go 1.27.1, C-компилятор для `-race` и Docker; для публичных проверок — Python 3.9+. На Windows используйте `PYTHON=python`. + +В Git Bash на Windows сначала выполните `export PYTHONUTF8=1`, чтобы Python читал UTF-8 контракты независимо от системной кодировки. В PowerShell эквивалент — `$env:PYTHONUTF8='1'`. + +```sh +cp .env.example .env +make secrets +make up +make test PYTHON=python +make check PYTHON=python +make unit +make vet +``` + +Полный зафиксированный Go-прогон: `go test -race -p 2 -count=1 ./...` (эквивалент `make unit GOFLAGS=-p=2`). Ограничена только параллельность пакетов с Docker-фикстурами; конкурентные тесты внутри пакетов не отключены. + +LB: `http://localhost:8080`; статистика HAProxy — только `127.0.0.1:8404`. Если порт занят, до запуска и restart-проверок задайте `export PUBLISH=127.0.0.1:18080 BASE_URL=http://localhost:18080`. + +После Compose `make up` и restart-hook дополнительно проверяют свежие readiness-ответы через LB: прежний Docker health status балансировщика может ещё оставаться healthy, пока его active checks выводят новые API из состояния DOWN. Ожидание ограничено 60 пробами, успех — три последовательных ответа. + +`make secrets` создаёт игнорируемый Git каталог `secrets/`; повторный запуск сохраняет существующие значения. Неполный комплект не заменяется молча. Секреты передаются файлами `*_FILE`, не значениями в `.env`. `make down` сохраняет volumes. **`make destroy` безвозвратно удаляет учебные данные PostgreSQL, Valkey и S3**. Удаление файлов секретов при сохранённых volumes не является сбросом паролей. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy → API × 2 → PostgreSQL (метаданные), Valkey (сессии), SeaweedFS (байты). Перенесены lab3 API, устойчивые сессии, идемпотентность сообщений, drain/rolling update и инструменты исследования. Серии старого трека 1 сохранены, их результаты не выдаются за lab4. + +SeaweedFS **4.53** и остальные Docker-образы закреплены digest в `compose.yaml`; SDK — `minio-go/v7` **7.4.1**. `weed server` хранит и filer-метаданные, и байты в `s3data`. Внутренние master/filer/volume HTTP и gRPC слушают loopback контейнера, исключая сетевой обход S3-авторизации. Наружу не опубликован ни S3, ни административный интерфейс. + +Миграция `00003_attachments.sql` добавляет: + +| Таблица | Назначение | +| --- | --- | +| `files(hash, size_bytes, mime_type, status, created_at)` | Одна запись на содержимое; PK SHA-256 — 32 байта `bytea`. `created_at` нужен для grace period. | +| `attachments(id, chat_id, owner_id, filename, file_hash, created_at)` | Логическое вложение с UUID и доступом через чат; существует до сообщения. | +| `message_attachments(chat_id, message_id, attachment_id, ordinal)` | Ссылки сообщения; `ordinal` сохраняет порядок при чтении и идемпотентном повторе. | + +`path` не хранится: ключ — `blobs/`. Имя используется только для отображения. Индекс `attachments(file_hash)` нужен для проверки использования общего файла; FK защищают ссылки. Принадлежность вложений чату проверяется одним запросом в транзакции отправки. История возвращает упорядоченные `attachment_ids` одним SQL-запросом. + +Одинаковые байты дают одну строку `files` и один постоянный объект. Каждая загрузка получает свой UUID, владельца, имя и чат: знание хэша не даёт доступа. Дедупликация экономит постоянное хранение, но не клиентский трафик — сервер должен принять тело, чтобы вычислить хэш. + +### Upload и download + +Принимаются сырые PNG/JPEG, **не multipart**, до **10 MiB** и **40 млн пикселей**, с истинным `Content-Type`. Нужен положительный `Content-Length`; неизвестная длина — 400. Имя — 1–255 code points, без разделителей пути и управляющих символов. Чужой объект даёт 404; аутентификация выполняется до разбора параметров. + +1. Поток с жёстким лимитом записывается в `staging/`, одновременно считаются размер и SHA-256. Локального оригинала на API нет. +2. Проверяются формат и число пикселей **до выделения пиксельного буфера**, затем изображение полностью декодируется для обнаружения повреждённого body. +3. Под транзакционной advisory-блокировкой хэша проверяются `files` и общий объект. Готовые байты не копируются повторно; отсутствующий blob восстанавливается из новой загрузки. +4. При необходимости серверный Copy публикует `blobs/`. Одна SQL-транзакция сохраняет готовый файл и новое логическое вложение; 201 возвращается после COMMIT. +5. Staging удаляется отдельной ограниченной по времени компенсацией; её неудачу покрывает reconciler. + +Пиксельный буфер 40 млн RGBA64 занимает 320 MB, но это не вся память декодера: progressive JPEG хранит коэффициенты, а PNG — текущую и предыдущую строки. Например, progressive JPEG 4:4:4 8000×5000 требует около 600 MB живых буферов, а однострочный RGBA64 PNG на 40 млн пикселей — около 960 MB. Поэтому допускается **один декодер на реплику**, API ограничен **2 GiB**, `GOMEMLIMIT=1536MiB`. Первоначальная оценка 512 MiB исправлена после независимого ревью. Upload/download остаются потоковыми; проверка только заголовка не используется, поскольку она пропускает повреждённый body. + +Обычный запрос, включая GET метаданных, имеет бюджет 5 s, бинарный — 30 s; socket и соответствующий HAProxy timeout — 35 s. Shutdown допускает 38 s на активные обработчики, Compose stop grace — 45 s с учётом drain 2 s. `MAX_UPLOAD_BYTES` и `MAX_IMAGE_PIXELS` можно уменьшить, но нельзя повысить сверх контракта. Для транзакций с S3 I/O локальный idle timeout увеличен до 35 s. Обрыв чтения S3 при декодировании возвращает 503; повреждённые байты, успешно прочитанные из хранилища, — 415. + +Download сначала авторизует вложение через чат, затем передаёт оригинал через API без redirect. `variants=[]`, `error_code=null`, у неизменяемой логической записи `updated_at=created_at`. Внутренний `failed` отвечает 409 вместо JSON вне обязательного enum lab4. + +S3 намеренно исключён из readiness: при его отказе файловые операции дают 503, текстовые сообщения и доступные метаданные продолжают работать. PostgreSQL и Valkey остаются критическими зависимостями. + +### Согласованность и reconciler + +Общей транзакции PostgreSQL/S3 нет: успешный Copy и неуспешный COMMIT оставляют orphan. При ошибке одной загрузки нельзя удалять общий blob — он может использоваться другими. Компенсируется только её staging. + +Публикация и удаление используют одну `pg_advisory_xact_lock` по первым 8 байтам SHA-256. Коллизия ключа блокировки лишь сериализует разные файлы. Перед изменением reconciler повторно проверяет метаданные, ссылки и возраст текущего объекта под этой блокировкой. Память обхода ограничена страницами до 500 записей. + +Grace period — 1 h, минимум 2 min, выше бюджета upload. Свежие данные и неизвестный возраст не разрешают удаление. По умолчанию CLI выполняет **dry-run без изменений**. + +| Находка после grace | Действие с `--apply` | +| --- | --- | +| Старый staging | Повторный HEAD, затем удаление. | +| Старый blob без `files` | Под блокировкой повторить проверку БД и возраста, удалить объект. | +| Старый pending/ready без единого логического вложения | Удалить объект, затем строку; свежие байты защищают даже старую строку. | +| Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. | +| Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. | + +Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready. + +```sh +make reconcile +make reconcile APPLY=1 +make reconcile DEEP=1 +``` + +В каждой API-реплике shallow apply выполняется каждые 10 min с grace 1 h и бюджетом прохода 2 min. Прерывание логируется; ручной CLI позволяет выполнить полный обход. Ошибка S3, запрет доступа или отсутствие bucket не считаются отсутствием отдельного файла. + +Bootstrap — одноразовый `s3-init` с отдельным admin-ключом; приватные статические identities загружаются до приёма запросов. После создания/проверки bucket bootstrap ждёт успешного чтения устойчивого служебного marker и write/read/delete-пробы: доступность S3 gateway сама по себе не означает, что прежние volumes уже зарегистрированы. Marker находится вне `staging/` и `blobs/`. Это барьер запуска, а не прозрачный retry загрузки пользователя. API имеет только Read/Write одного bucket, reconciler — дополнительно List. `S3_RECONCILE_*_FILE` выбирают отдельные ключи фонового обхода. `SESSION_STORE=postgres` и `CACHE_URL_FILE` в исходном skeleton не используются: фактический store сессий — Valkey через `SESSION_STORE_URL_FILE`, отдельного кэша нет. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Обязательные `tests/`, `contracts/openapi.json` и README задания сохранены. Собственные проверки используют настоящий PostgreSQL/SeaweedFS; узкий двойник байтов нужен для детерминированных отказов. Отказ после Copy также внедряется SQL trigger, отвергающим INSERT метаданных. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / доказательство | +| --- | --- | +| Smoke: checksum, сообщение, ACL | `make test`; `evidence/smoke.txt` | +| Restart БД/S3/сессий с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd PYTHON=python`; `evidence/persistence.txt` | +| Failover загрузившей реплики | `python -X utf8 scripts/lab4/failover.py`; `evidence/failover-uploaders.json`, обе реплики | +| Конкурентная запись при отказе | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd PYTHON=python`; `evidence/failover.txt` | +| MIME, повреждение, bytes/pixels, имя, чужой чат | `internal/attachment/validation_test.go`, `internal/app/attachment_test.go` | +| Дедупликация, гонка upload/GC, SQL rollback | `internal/attachment/service_test.go` | +| Приватность bucket вне API | `evidence/private-bucket.txt` | +| Credentials, bootstrap, S3 restart | `internal/platform/s3store/*_test.go`, `internal/testenv/s3_test.go` | +| Отказ S3, прерывание upload | `TestAttachmentRealS3RoundTripValidationAndOutage` | +| Полный race/vet/check | `make unit`, `make vet`, `make check`; `evidence/tests.txt` | +| ≥10 одновременных upload, память API | `make upload-memory PYTHON=python`; `evidence/upload-memory.json` | +| Трек 2: faults, dry/apply/repeat, стоимость | `go run ./tools/reconcilebench`; `evidence/track2/` | + +На Linux используйте `.sh` hooks; на Windows `.cmd` вызывает Git Bash `sh`, поскольку Python запускает hook без shell interpolation. Failover-hook оставляет выбранную реплику остановленной: восстановите её до повторного теста. + +Публичная persistence-проверка прошла за 57.838 s: сохранены сообщения, исходный файл и прежняя сессия; `StartedAt` в доказательстве подтверждает перезапуск API, LB, PostgreSQL, Valkey и S3. Собственный failover остановил именно реплику из ответа upload для каждого из двух API: 4.837/4.160 s на остановку, после неё другой API вернул те же байты. Публичный concurrent failover: 130 подтверждённых записей, 0 неудачных попыток, восстановление записи за 4.265 s; дублей и потери подтверждённых данных нет. Остановленные API восстановлены. + +Дополнительный `make validate` требует `requirements-dev.txt`. В этой сессии установка валидатора из PyPI не завершилась из-за сетевых timeout, поэтому локальная полная OpenAPI-валидация не заявляется. Контракт не изменён; offline check и используемое публичными тестами подмножество JSON Schema проверены. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза трека 2: сериализация по хэшу и повторные проверки позволяют безопасно убирать старые orphan/pending, сохраняя свежие и используемые данные; повторный проход не выполняет повторных исправлений. Deep дополнительно читает байты и потому дороже shallow. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Машинные исходные данные и методика находятся в `evidence/track2/` и `evidence/upload-memory.json`. Память измеряется у контейнеров API. Семплирование может пропустить короткий пик: это наблюдённый максимум Docker working set, не доказательство абсолютного RSS bound. + +Два burst-прогона по 10 одновременно стартующих загрузок на API × 2, baseline/tail по 3 s, одинаковые байты внутри серии, прогретое хранилище. Дедупликация могла использовать объект предыдущего прогона, но каждое тело полностью передавалось и декодировалось. Во всех 20 запросах — 201 и правильный checksum; последующее скачивание каждой серии совпало побайтно. + +| PNG payload | Успехи | p50, ms | p95/max, ms | Наибольший sampled working set реплики, MiB | +| --- | ---: | ---: | ---: | ---: | +| 1 MiB, 512×512 RGB | 10/10 | 216.34 | 259.15 | 69.6 | +| 10 MiB, 1600×1600 RGB | 10/10 | 1 865.39 | 1 997.06 | 69.6 | + +При 10 наблюдениях nearest-rank p95 совпадает с максимумом, это не устойчивый хвостовой percentile. Маленькая серия короче частоты семплирования; post-tail сохранён, но краткие пики могут быть пропущены. Это эксперимент памяти, не измерение точки насыщения или production throughput. + +Отдельно выполнены одиночные upload/download на границе **40 млн пикселей** с более дорогими путями декодера: + +| Формат | Размер входа, B | Наибольший sampled working set, MiB | Итог | +| --- | ---: | ---: | --- | +| Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart | +| Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart | +| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart | + +Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки. + +Для воспроизведения тяжёлых fixtures нужен только дополнительный **Pillow 11.1.0** в своём Python-окружении; основной API и harness от него не зависят. Генератор пишет в игнорируемый `bin/`, а не в обязательные публичные fixtures: + +```sh +python -X utf8 scripts/lab4/generate-decoder-fixtures.py +python -X utf8 scripts/lab4/decoder-memory.py --file bin/decoder-fixtures/progressive-rgb-444-40mp.jpg --mime image/jpeg --label progressive-rgb --output evidence/decoder-memory-progressive-rgb.json +python -X utf8 scripts/lab4/decoder-memory.py --file bin/decoder-fixtures/progressive-cmyk-444-40mp.jpg --mime image/jpeg --label progressive-cmyk --output evidence/decoder-memory-progressive-cmyk.json +python -X utf8 scripts/lab4/decoder-memory.py --file bin/decoder-fixtures/wide-rgba64-40mp.png --mime image/png --label wide-rgba64 --output evidence/decoder-memory-wide-rgba64.json +``` + +Среда: Windows amd64, 16 логических CPU у клиента, Docker VM — 8 CPU и 12 547 280 896 B RAM. Лимиты Compose: API по 1 CPU/2 GiB, PostgreSQL 1 CPU/512 MiB, S3 1 CPU/512 MiB, HAProxy 0.5 CPU/128 MiB, Valkey 0.5 CPU/320 MiB. Данные синтетические; общий host не является выделенным нагрузочным стендом. + +Трек 2: использованы отдельные реальные PostgreSQL/S3 testcontainers, grace 120 s и реальное ожидание 125 s перед очисткой старых объектов. Dry-run нашёл 6 несогласованностей и не изменил данные. Apply выполнил 5 исправлений; сообщение о намеренно повреждённом используемом объекте осталось без удаления. Повторный apply выполнил **0 исправлений**. Проверки сохранили свежие staging/orphan/pending, ссылки сообщений и используемые объекты. + +| Используемых файлов/объектов | Shallow dry-run, ms | SQL statements | Операции порта S3 | Sampled Go HeapAlloc, B | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 184 | 2 | 103 | 3 875 232 | +| 1 000 | 2 160 | 3 | 1 004 | 4 159 720 | +| 5 000 | 10 121 | 11 | 5 012 | 5 157 552 | + +Это один прогон на размер на прогретом стенде, без статистических повторов. Измерена память Go-процесса сверки во время Run, **не RSS и не память PostgreSQL/S3**. В стоимости S3 считаются вызовы порта, а не все внутренние HTTP-запросы SDK. Для cost-датасета SQL-метаданные старые, реальные объекты свежие: дополнительная проверка orphan для них ещё не нужна. Отдельный regression на 1 003 старых объектах проверяет пакетное чтение БД. 10k/50k и production SLA не заявляются. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- PostgreSQL, Valkey, SeaweedFS, LB и Docker host остаются единичными точками отказа; отказ API-реплики не доказывает HA всего стенда. +- SHA-256 используется в принятой модели криптографических коллизий. Дедупликация не позволяет клиенту пропустить отправку тела. +- Удаление клиентских вложений и сборка «давно не прикреплённых» логических вложений не входят в контракт. +- Повреждение используемого объекта обнаруживается deep-проверкой, но исходные байты нельзя восстановить без новой загрузки или backup. +- Между страницами сверки нет snapshot; безопасность изменений обеспечивают повторные проверки под блокировкой. +- Периодический обход ограничен двумя минутами и начинает каждый проход заново: для существенно больших наборов нужен ручной полный CLI-проход либо будущий resumable cursor. 5 000 объектов укладываются в измеренный бюджет; автоматическое продвижение на произвольном объёме не доказано. +- После аварии bootstrap и одновременно неудачной компенсации может остаться небольшой служебный `.health/check-*`; это не вложение, и attachment-reconciler не обходит служебный namespace. Постоянный `.health/storage-v1` удалять нельзя. +- HTTP предназначен для локального изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. + +## Источники + +Контракт текущего репозитория; [SeaweedFS](https://github.com/seaweedfs/seaweedfs), [minio-go](https://github.com/minio/minio-go), [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [PostgreSQL advisory locks](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS). + +ИИ-ассистент использовался как парный программист: генерация кода, тестов и черновиков документации по моим архитектурным решениям с последующим ревью каждого модуля и переработкой по замечаниям; выбор стека, структура проекта, модель данных, обработка ошибок, логирование и границы модулей определены мной; все проверки воспроизведены мной вручную. -- 2.54.0 From 1f06d9f4abc02ca41adb697107b7d3879536a145 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:40:17 +0300 Subject: [PATCH 5/6] =?UTF-8?q?=D0=A1=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=82=D0=B5=20=D0=B4=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=81=D1=82=D0=B2=D0=B0=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=BE=D0=BA=20=D0=B8=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2=20=D0=BB?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD=D0=BE?= =?UTF-8?q?=D0=B9=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evidence/decoder-memory-progressive-cmyk.json | 764 +++++++++ evidence/decoder-memory-progressive-rgb.json | 655 ++++++++ evidence/decoder-memory-wide-rgba64.json | 613 +++++++ evidence/environment.json | 22 + evidence/failover-uploaders.json | 96 ++ evidence/failover.txt | 23 + evidence/persistence.txt | 25 + evidence/private-bucket.txt | 6 + evidence/review.txt | 23 + evidence/smoke.txt | 2 + evidence/tests.txt | 34 + evidence/track2/live-apply.jsonl | 1 + evidence/track2/live-dry.jsonl | 1 + evidence/track2/live-repeat.jsonl | 1 + evidence/track2/results.json | 193 +++ evidence/upload-memory-initial-512MiB.json | 1447 ++++++++++++++++ evidence/upload-memory.json | 1486 +++++++++++++++++ 17 files changed, 5392 insertions(+) create mode 100644 evidence/decoder-memory-progressive-cmyk.json create mode 100644 evidence/decoder-memory-progressive-rgb.json create mode 100644 evidence/decoder-memory-wide-rgba64.json create mode 100644 evidence/environment.json create mode 100644 evidence/failover-uploaders.json create mode 100644 evidence/failover.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/private-bucket.txt create mode 100644 evidence/review.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/tests.txt create mode 100644 evidence/track2/live-apply.jsonl create mode 100644 evidence/track2/live-dry.jsonl create mode 100644 evidence/track2/live-repeat.jsonl create mode 100644 evidence/track2/results.json create mode 100644 evidence/upload-memory-initial-512MiB.json create mode 100644 evidence/upload-memory.json diff --git a/evidence/decoder-memory-progressive-cmyk.json b/evidence/decoder-memory-progressive-cmyk.json new file mode 100644 index 00000000..c64470b4 --- /dev/null +++ b/evidence/decoder-memory-progressive-cmyk.json @@ -0,0 +1,764 @@ +{ + "schema_version": 1, + "name": "progressive-cmyk", + "mime": "image/jpeg", + "fixture": { + "width": 8000, + "height": 5000, + "bit_depth": 8, + "components": 4, + "progressive": true, + "sampling": [ + 17, + 17, + 17, + 17 + ] + }, + "payload_bytes": 939079, + "sha256": "0b38b5e95ca2623f585bae77cdfba9259a6b786971e6b9b8d10d774177023956", + "requests": 1, + "ok": true, + "started_at": "2026-11-16T09:16:29.819614+00:00", + "measurement": { + "source": "docker stats JSON stream, formatted working-set values", + "limitation": "sampled/rounded peaks can miss instantaneous maxima", + "baseline_seconds": 3, + "tail_seconds": 3 + }, + "api_limits": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": 2147483648, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": 2147483648 + }, + "state_before": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "duration_s": 2.580977499950677, + "memory_samples": [ + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 0.549924, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 0.549959, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 0.549984, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 0.549996, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 1.049104, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 1.049131, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 1.049142, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 1.049151, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73190605, + "limit_bytes": 2147483648, + "elapsed_s": 1.548806, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 1.548828, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73190605, + "limit_bytes": 2147483648, + "elapsed_s": 1.548839, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 1.548847, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73190605, + "limit_bytes": 2147483648, + "elapsed_s": 2.049053, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 2.049071, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73190605, + "limit_bytes": 2147483648, + "elapsed_s": 2.04908, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 2.049087, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 2.558386, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 2.558406, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 2.558415, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 2.558422, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 3.048888, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 3.04891, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73326920, + "limit_bytes": 2147483648, + "elapsed_s": 3.04892, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 3.048928, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73452749, + "limit_bytes": 2147483648, + "elapsed_s": 3.548966, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 3.54899, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73452749, + "limit_bytes": 2147483648, + "elapsed_s": 3.548999, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 3.549006, + "case": "progressive-cmyk", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73452749, + "limit_bytes": 2147483648, + "elapsed_s": 4.048986, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 4.049008, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 73452749, + "limit_bytes": 2147483648, + "elapsed_s": 4.049019, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 4.049027, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 620652134, + "limit_bytes": 2147483648, + "elapsed_s": 4.548787, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 4.548811, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 620652134, + "limit_bytes": 2147483648, + "elapsed_s": 4.548823, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 4.548832, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 620652134, + "limit_bytes": 2147483648, + "elapsed_s": 5.049002, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 5.049024, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 620652134, + "limit_bytes": 2147483648, + "elapsed_s": 5.049034, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 5.049042, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 792199168, + "limit_bytes": 2147483648, + "elapsed_s": 5.549043, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 5.549064, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 792199168, + "limit_bytes": 2147483648, + "elapsed_s": 5.549074, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 5.549086, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 792199168, + "limit_bytes": 2147483648, + "elapsed_s": 6.048631, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 6.048659, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 792199168, + "limit_bytes": 2147483648, + "elapsed_s": 6.048671, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 6.048682, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991428608, + "limit_bytes": 2147483648, + "elapsed_s": 6.549034, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 6.549058, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991428608, + "limit_bytes": 2147483648, + "elapsed_s": 6.549069, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 6.549079, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991428608, + "limit_bytes": 2147483648, + "elapsed_s": 7.048895, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 7.048919, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991428608, + "limit_bytes": 2147483648, + "elapsed_s": 7.048931, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 7.048941, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 7.548935, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 7.548953, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 7.548963, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 7.54897, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 8.048712, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 8.048734, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 8.048748, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653472563, + "limit_bytes": 2147483648, + "elapsed_s": 8.048756, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 8.549399, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 8.549422, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 8.549433, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 8.549442, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 9.049432, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 9.049453, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 991323750, + "limit_bytes": 2147483648, + "elapsed_s": 9.049463, + "case": "progressive-cmyk", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 653577421, + "limit_bytes": 2147483648, + "elapsed_s": 9.049471, + "case": "progressive-cmyk", + "phase": "load" + } + ], + "sampling_error": null, + "state_after": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "memory": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "baseline_bytes": 73452749, + "sampled_peak_bytes": 991428608, + "delta_bytes": 917975859, + "baseline_samples": 14, + "load_samples": 22 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "baseline_bytes": 653577421, + "sampled_peak_bytes": 653577421, + "delta_bytes": 0, + "baseline_samples": 14, + "load_samples": 22 + } + }, + "observed_max_inflight": 1, + "accepted": 1, + "status_counts": { + "201": 1 + }, + "latency": { + "p50_ms": 2580.686, + "p95_ms": 2580.686, + "max_ms": 2580.686 + }, + "attempted_bytes": 939079, + "successful_bytes": 939079, + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 939079, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-da4a-7ddc-8109-d9ac8efc99b7", + "latency_ms": 2580.686, + "start_offset_ms": 0.032 + } + ], + "download": { + "ok": true, + "status": 200, + "bytes": 939079, + "sha256": "0b38b5e95ca2623f585bae77cdfba9259a6b786971e6b9b8d10d774177023956" + } +} diff --git a/evidence/decoder-memory-progressive-rgb.json b/evidence/decoder-memory-progressive-rgb.json new file mode 100644 index 00000000..ef50cb6b --- /dev/null +++ b/evidence/decoder-memory-progressive-rgb.json @@ -0,0 +1,655 @@ +{ + "schema_version": 1, + "name": "progressive-rgb", + "mime": "image/jpeg", + "fixture": { + "width": 8000, + "height": 5000, + "bit_depth": 8, + "components": 3, + "progressive": true, + "sampling": [ + 17, + 17, + 17 + ] + }, + "payload_bytes": 469711, + "sha256": "7c01cae2900b154361df76733c19db0737e19fe1c3bd0b9fae82595b7fbf4fad", + "requests": 1, + "ok": true, + "started_at": "2026-11-16T09:16:19.759532+00:00", + "measurement": { + "source": "docker stats JSON stream, formatted working-set values", + "limitation": "sampled/rounded peaks can miss instantaneous maxima", + "baseline_seconds": 3, + "tail_seconds": 3 + }, + "api_limits": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": 2147483648, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": 2147483648 + }, + "state_before": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "duration_s": 1.4607699000043795, + "memory_samples": [ + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 1.547282, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70894223, + "limit_bytes": 2147483648, + "elapsed_s": 1.547322, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 1.547345, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70894223, + "limit_bytes": 2147483648, + "elapsed_s": 1.547356, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 2.047578, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70894223, + "limit_bytes": 2147483648, + "elapsed_s": 2.047641, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 2.047659, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70894223, + "limit_bytes": 2147483648, + "elapsed_s": 2.04767, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 2.547018, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71030538, + "limit_bytes": 2147483648, + "elapsed_s": 2.547037, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 2.547046, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71030538, + "limit_bytes": 2147483648, + "elapsed_s": 2.547053, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 3.046578, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71030538, + "limit_bytes": 2147483648, + "elapsed_s": 3.046597, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 3.046605, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71030538, + "limit_bytes": 2147483648, + "elapsed_s": 3.046612, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 3.546984, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71156367, + "limit_bytes": 2147483648, + "elapsed_s": 3.547006, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 3.547017, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71156367, + "limit_bytes": 2147483648, + "elapsed_s": 3.547025, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 4.046531, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71156367, + "limit_bytes": 2147483648, + "elapsed_s": 4.046552, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 4.046562, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71156367, + "limit_bytes": 2147483648, + "elapsed_s": 4.04657, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 4.546655, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71187825, + "limit_bytes": 2147483648, + "elapsed_s": 4.546682, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 4.546693, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71187825, + "limit_bytes": 2147483648, + "elapsed_s": 4.546701, + "case": "progressive-rgb", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 5.04708, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71187825, + "limit_bytes": 2147483648, + "elapsed_s": 5.047106, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 5.047121, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 71187825, + "limit_bytes": 2147483648, + "elapsed_s": 5.047132, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 5.546886, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 578499379, + "limit_bytes": 2147483648, + "elapsed_s": 5.546906, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 5.546918, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 578499379, + "limit_bytes": 2147483648, + "elapsed_s": 5.546926, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 6.047261, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 578499379, + "limit_bytes": 2147483648, + "elapsed_s": 6.047287, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 56927191, + "limit_bytes": 2147483648, + "elapsed_s": 6.047298, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 578499379, + "limit_bytes": 2147483648, + "elapsed_s": 6.047307, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 6.547209, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 6.547231, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 6.547247, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 6.547255, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 7.047074, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 7.047093, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 7.047102, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 7.047109, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57189335, + "limit_bytes": 2147483648, + "elapsed_s": 7.546834, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 7.546855, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57189335, + "limit_bytes": 2147483648, + "elapsed_s": 7.546864, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 7.546872, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57189335, + "limit_bytes": 2147483648, + "elapsed_s": 8.047107, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 8.047126, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57189335, + "limit_bytes": 2147483648, + "elapsed_s": 8.047135, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613626675, + "limit_bytes": 2147483648, + "elapsed_s": 8.047142, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 8.547593, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613521818, + "limit_bytes": 2147483648, + "elapsed_s": 8.547632, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57053020, + "limit_bytes": 2147483648, + "elapsed_s": 8.547646, + "case": "progressive-rgb", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 613521818, + "limit_bytes": 2147483648, + "elapsed_s": 8.547655, + "case": "progressive-rgb", + "phase": "load" + } + ], + "sampling_error": null, + "state_after": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "memory": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "baseline_bytes": 56927191, + "sampled_peak_bytes": 57189335, + "delta_bytes": 262144, + "baseline_samples": 14, + "load_samples": 16 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "baseline_bytes": 71187825, + "sampled_peak_bytes": 613626675, + "delta_bytes": 542438850, + "baseline_samples": 14, + "load_samples": 16 + } + }, + "observed_max_inflight": 1, + "accepted": 1, + "status_counts": { + "201": 1 + }, + "latency": { + "p50_ms": 1460.381, + "p95_ms": 1460.381, + "max_ms": 1460.381 + }, + "attempted_bytes": 469711, + "successful_bytes": 469711, + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 469711, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-b65d-7795-aaf3-911e43a58f12", + "latency_ms": 1460.381, + "start_offset_ms": 0.021 + } + ], + "download": { + "ok": true, + "status": 200, + "bytes": 469711, + "sha256": "7c01cae2900b154361df76733c19db0737e19fe1c3bd0b9fae82595b7fbf4fad" + } +} diff --git a/evidence/decoder-memory-wide-rgba64.json b/evidence/decoder-memory-wide-rgba64.json new file mode 100644 index 00000000..d5fccadb --- /dev/null +++ b/evidence/decoder-memory-wide-rgba64.json @@ -0,0 +1,613 @@ +{ + "schema_version": 1, + "name": "wide-rgba64", + "mime": "image/png", + "fixture": { + "width": 40000000, + "height": 1, + "bit_depth": 16, + "color_type": 6 + }, + "payload_bytes": 1397354, + "sha256": "25d5a4133d85d830fc5d7e5946a694cd71b673bc83acc362ec097f2c601b96e2", + "requests": 1, + "ok": true, + "started_at": "2026-11-16T09:16:40.211157+00:00", + "measurement": { + "source": "docker stats JSON stream, formatted working-set values", + "limitation": "sampled/rounded peaks can miss instantaneous maxima", + "baseline_seconds": 3, + "tail_seconds": 3 + }, + "api_limits": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": 2147483648, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": 2147483648 + }, + "state_before": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "duration_s": 0.630827400018461, + "memory_samples": [ + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034420224, + "limit_bytes": 2147483648, + "elapsed_s": 1.553241, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 1.553274, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034420224, + "limit_bytes": 2147483648, + "elapsed_s": 1.553287, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 1.553295, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034420224, + "limit_bytes": 2147483648, + "elapsed_s": 2.055202, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 2.055222, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034420224, + "limit_bytes": 2147483648, + "elapsed_s": 2.05524, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 2.055258, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 2.553574, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 2.553593, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 2.553602, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 2.55365, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 3.05266, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 3.052681, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 3.052691, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 3.052699, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1038090240, + "limit_bytes": 2147483648, + "elapsed_s": 3.553008, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 3.553026, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1038090240, + "limit_bytes": 2147483648, + "elapsed_s": 3.553034, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 3.553041, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1038090240, + "limit_bytes": 2147483648, + "elapsed_s": 4.052797, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 4.052817, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1038090240, + "limit_bytes": 2147483648, + "elapsed_s": 4.052825, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 4.052832, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 4.55255, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713555968, + "limit_bytes": 2147483648, + "elapsed_s": 4.552569, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 4.552581, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713555968, + "limit_bytes": 2147483648, + "elapsed_s": 4.552588, + "case": "wide-rgba64", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 5.052981, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713555968, + "limit_bytes": 2147483648, + "elapsed_s": 5.053002, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1034525082, + "limit_bytes": 2147483648, + "elapsed_s": 5.053012, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713555968, + "limit_bytes": 2147483648, + "elapsed_s": 5.05302, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1264867869, + "limit_bytes": 2147483648, + "elapsed_s": 5.552655, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 5.552674, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1264867869, + "limit_bytes": 2147483648, + "elapsed_s": 5.552682, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 5.552688, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1264867869, + "limit_bytes": 2147483648, + "elapsed_s": 6.052855, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 6.052874, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1264867869, + "limit_bytes": 2147483648, + "elapsed_s": 6.052882, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 6.052889, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 6.552602, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 6.552622, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 6.552632, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 6.552644, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 7.052492, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 7.052511, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 7.05252, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 7.052528, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 7.553232, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 7.553251, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 7.55326, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 7.553268, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 8.053203, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 8.053222, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 1352914698, + "limit_bytes": 2147483648, + "elapsed_s": 8.05323, + "case": "wide-rgba64", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 713451110, + "limit_bytes": 2147483648, + "elapsed_s": 8.053237, + "case": "wide-rgba64", + "phase": "load" + } + ], + "sampling_error": null, + "state_after": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "memory": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "baseline_bytes": 1034525082, + "sampled_peak_bytes": 1352914698, + "delta_bytes": 318389616, + "baseline_samples": 14, + "load_samples": 14 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "baseline_bytes": 713555968, + "sampled_peak_bytes": 713555968, + "delta_bytes": 0, + "baseline_samples": 14, + "load_samples": 14 + } + }, + "observed_max_inflight": 1, + "accepted": 1, + "status_counts": { + "201": 1 + }, + "latency": { + "p50_ms": 630.558, + "p95_ms": 630.558, + "max_ms": 630.558 + }, + "attempted_bytes": 1397354, + "successful_bytes": 1397354, + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 1397354, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a5-0702-7b0a-8650-57d4e66f7c31", + "latency_ms": 630.558, + "start_offset_ms": 0.02 + } + ], + "download": { + "ok": true, + "status": 200, + "bytes": 1397354, + "sha256": "25d5a4133d85d830fc5d7e5946a694cd71b673bc83acc362ec097f2c601b96e2" + } +} diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..f6b16f9b --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,22 @@ +{ + "collected_at": "2026-11-16T09:38:59.955076+00:00", + "source_code_commit": "29f45ca0eb0ca060f77a8ada1e20aeca4d7e0785", + "go": "go version go1.27.1 windows/amd64", + "docker_version": "28.5.1", + "docker_cpus": 8, + "docker_memory_bytes": 12547280896, + "driver_system": "Windows", + "driver_arch": "AMD64", + "driver_logical_cpus": 16, + "python": "3.13.1", + "api_image": "sha256:5f84776e9db8de84b736cf9a58332830042618c6ea4a2097640a9242cc5a660e", + "running_services": [ + "lab4-api-1 running healthy", + "lab4-api-2 running healthy", + "lab4-lb-1 running healthy", + "lab4-postgres-1 running healthy", + "lab4-s3-1 running healthy", + "lab4-valkey-1 running healthy" + ], + "scope": "final source snapshot; individual experiment inputs, dates and metric definitions are in their own evidence files" +} diff --git a/evidence/failover-uploaders.json b/evidence/failover-uploaders.json new file mode 100644 index 00000000..13894096 --- /dev/null +++ b/evidence/failover-uploaders.json @@ -0,0 +1,96 @@ +{ + "scenario": "stop actual upload-serving replica; download via survivor", + "cases": [ + { + "uploader": { + "container": "e0755982c2d3", + "replica": "lab4-api-1", + "state": "running", + "alias": "1326e1bfb6754455", + "image": "5f84776e9db8" + }, + "download_instance": "4cfce35bab4eba13", + "stop_seconds": 4.837, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "same_bytes": true, + "before": [ + { + "container": "e0755982c2d3", + "replica": "lab4-api-1", + "state": "running", + "alias": "1326e1bfb6754455", + "image": "5f84776e9db8" + }, + { + "container": "4705eb662d22", + "replica": "lab4-api-2", + "state": "running", + "alias": "4cfce35bab4eba13", + "image": "5f84776e9db8" + } + ], + "after": [ + { + "container": "e0755982c2d3", + "replica": "lab4-api-1", + "state": "exited", + "alias": "-", + "image": "5f84776e9db8" + }, + { + "container": "4705eb662d22", + "replica": "lab4-api-2", + "state": "running", + "alias": "4cfce35bab4eba13", + "image": "5f84776e9db8" + } + ] + }, + { + "uploader": { + "container": "4705eb662d22", + "replica": "lab4-api-2", + "state": "running", + "alias": "4cfce35bab4eba13", + "image": "5f84776e9db8" + }, + "download_instance": "1caa70dbe06803f5", + "stop_seconds": 4.16, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "same_bytes": true, + "before": [ + { + "container": "e0755982c2d3", + "replica": "lab4-api-1", + "state": "running", + "alias": "1caa70dbe06803f5", + "image": "5f84776e9db8" + }, + { + "container": "4705eb662d22", + "replica": "lab4-api-2", + "state": "running", + "alias": "4cfce35bab4eba13", + "image": "5f84776e9db8" + } + ], + "after": [ + { + "container": "e0755982c2d3", + "replica": "lab4-api-1", + "state": "running", + "alias": "1caa70dbe06803f5", + "image": "5f84776e9db8" + }, + { + "container": "4705eb662d22", + "replica": "lab4-api-2", + "state": "exited", + "alias": "-", + "image": "5f84776e9db8" + } + ] + } + ], + "ok": true +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..7316493e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,23 @@ +Public lab4 failover with concurrent idempotent messages and retained attachment. +Before: +lab4-api-1 running +lab4-api-2 running +lab4-lb-1 running +lab4-migrate-1 exited +lab4-postgres-1 running +lab4-s3-1 running +lab4-s3-init-1 exited +lab4-valkey-1 running +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 4.265} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +After: +lab4-api-1 running +lab4-api-2 exited +lab4-lb-1 running +lab4-migrate-1 exited +lab4-postgres-1 running +lab4-s3-1 running +lab4-s3-init-1 exited +lab4-valkey-1 running +Elapsed seconds: 16.326 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..f5671952 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,25 @@ +Command: python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd +Before: +/lab4-api-1 started=2026-11-16T09:12:00.227764693Z status=running +/lab4-api-2 started=2026-11-16T09:12:00.497951153Z status=running +/lab4-lb-1 started=2026-11-16T08:30:34.254973105Z status=running +/lab4-migrate-1 started=2026-11-16T09:11:59.49568462Z status=exited +/lab4-postgres-1 started=2026-11-16T08:30:34.24518074Z status=running +/lab4-s3-1 started=2026-11-16T08:30:34.241553958Z status=running +/lab4-s3-init-1 started=2026-11-16T09:11:59.509084337Z status=exited +/lab4-valkey-1 started=2026-11-16T08:30:34.223057181Z status=running + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +After: +/lab4-api-1 started=2026-11-16T09:31:00.646703947Z status=running +/lab4-api-2 started=2026-11-16T09:31:00.927490539Z status=running +/lab4-lb-1 started=2026-11-16T09:30:59.396418626Z status=running +/lab4-migrate-1 started=2026-11-16T09:30:59.901158645Z status=exited +/lab4-postgres-1 started=2026-11-16T09:30:30.351009425Z status=running +/lab4-s3-1 started=2026-11-16T09:30:30.357773314Z status=running +/lab4-s3-init-1 started=2026-11-16T09:30:59.909286202Z status=exited +/lab4-valkey-1 started=2026-11-16T09:30:30.342892671Z status=running + +Elapsed seconds: 57.838 diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..9493db7a --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,6 @@ +Direct unauthenticated GET of existing private object from API container: +Connecting to s3:8333 (172.18.0.3:8333) + HTTP/1.1 403 Forbidden +wget: server returned error: HTTP/1.1 403 Forbidden + +PASS: no bytes available without S3 credentials. diff --git a/evidence/review.txt b/evidence/review.txt new file mode 100644 index 00000000..aa17e69e --- /dev/null +++ b/evidence/review.txt @@ -0,0 +1,23 @@ +Technical review completed after implementation and a scoped fix verification. + +Resolved: +- Maximum-pixel progressive JPEG / wide RGBA64 PNG required more than the + original 512 MiB estimate. API limit is 2 GiB, GOMEMLIMIT 1536MiB, one + decoder per replica. Three valid 40MP formats were accepted without OOM. +- Shutdown default 38s and Compose grace 45s cover active binary handlers. + Real upload and download continued for 11.5s across shutdown; race tests pass. +- JSON attachment metadata uses the normal 5s deadline, not the binary 30s + group. Connection-pool contention regression passes. +- Mid-stream S3 reader faults preserve dependency-error classification (503), + while malformed bytes successfully read from storage retain 415. +- S3 bootstrap waits for retained-byte routing and a write/read/delete probe. + Three consecutive container replacements preserve exact bytes. + +No P1/P2 blockers remain in the reviewed feature scope. Complete Go race, +vet/check, public smoke, full-stack persistence and both actual uploader +replica failures have saved passing evidence. + +Declared nonblocking limits: tiny abandoned startup probes may remain after +failed cleanup; the two-minute periodic pass does not prove progress through +arbitrarily large data. Use the full CLI scan for larger datasets. Working-set +and Go-heap measurements are sampled, distinct metrics, not universal maxima. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..7f15722c --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +Command: python -X utf8 tests/smoke.py --base-url http://localhost:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/tests.txt b/evidence/tests.txt new file mode 100644 index 00000000..c9ca175d --- /dev/null +++ b/evidence/tests.txt @@ -0,0 +1,34 @@ +Command: go test -race -p 2 -count=1 ./... +? messenger/cmd/migrate [no test files] +? messenger/cmd/reconcile [no test files] +? messenger/cmd/s3init [no test files] +ok messenger/cmd/server 6.787s +ok messenger/internal/app 45.607s +ok messenger/internal/attachment 49.580s +? messenger/internal/attachment/store [no test files] +ok messenger/internal/auth 7.745s +? messenger/internal/auth/store [no test files] +ok messenger/internal/chat 8.943s +ok messenger/internal/chat/store 8.784s +? messenger/internal/health [no test files] +ok messenger/internal/platform/apperr 1.550s +ok messenger/internal/platform/config 1.721s +ok messenger/internal/platform/httpx 1.220s +ok messenger/internal/platform/ident 1.543s +ok messenger/internal/platform/pagination 1.622s +ok messenger/internal/platform/password 2.356s +ok messenger/internal/platform/postgres 5.178s +ok messenger/internal/platform/s3store 205.312s +ok messenger/internal/platform/valkey 2.276s +ok messenger/internal/server 18.093s +ok messenger/internal/testenv 2.536s +ok messenger/internal/user 4.805s +? messenger/internal/user/store [no test files] +? messenger/migrations [no test files] +ok messenger/tools/concurrency 6.197s +ok messenger/tools/loadgen 10.987s +ok messenger/tools/reconcilebench 1.205s +PASS: go vet ./... +PASS: python -X utf8 scripts/check_contract.py +PASS: 13 Python experiment harness tests +PASS: regenerated 40MP fixtures match measured SHA-256 values diff --git a/evidence/track2/live-apply.jsonl b/evidence/track2/live-apply.jsonl new file mode 100644 index 00000000..001150ed --- /dev/null +++ b/evidence/track2/live-apply.jsonl @@ -0,0 +1 @@ +{"elapsed_ms":10,"sql_queries":8,"s3_calls":4,"scanned_files":6,"scanned_objects":6,"findings":0,"repaired":0} diff --git a/evidence/track2/live-dry.jsonl b/evidence/track2/live-dry.jsonl new file mode 100644 index 00000000..8d3aa45e --- /dev/null +++ b/evidence/track2/live-dry.jsonl @@ -0,0 +1 @@ +{"elapsed_ms":10,"sql_queries":3,"s3_calls":4,"scanned_files":6,"scanned_objects":6,"findings":0,"repaired":0} diff --git a/evidence/track2/live-repeat.jsonl b/evidence/track2/live-repeat.jsonl new file mode 100644 index 00000000..319fcf76 --- /dev/null +++ b/evidence/track2/live-repeat.jsonl @@ -0,0 +1 @@ +{"elapsed_ms":14,"sql_queries":8,"s3_calls":4,"scanned_files":6,"scanned_objects":6,"findings":0,"repaired":0} diff --git a/evidence/track2/results.json b/evidence/track2/results.json new file mode 100644 index 00000000..7e0e01db --- /dev/null +++ b/evidence/track2/results.json @@ -0,0 +1,193 @@ +[ + { + "grace_ms": 120000, + "memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3", + "phase": "environment", + "postgres_image": "postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873", + "s3_image": "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0", + "wait_ms": 125000 + }, + { + "phase": "fault", + "mode": "dry", + "target_files": 7, + "go_heap_peak_bytes": 2621072, + "stats": { + "elapsed_ms": 27, + "sql_queries": 3, + "s3_calls": 12, + "scanned_files": 7, + "scanned_objects": 9, + "findings": 6, + "repaired": 0 + }, + "findings": [ + { + "kind": "stale_staging", + "key": "staging/old", + "action": "delete_object", + "applied": false + }, + { + "kind": "orphan_blob", + "key": "blobs/ffea09ba14e194c3863fd5338b4994073275de7f1cfb799a4b82cf27340753b3", + "action": "delete_object", + "applied": false + }, + { + "kind": "unreferenced_ready", + "key": "blobs/0827f495ee64051e55f98e1d89fa78f6e85df2a36c08745dd73b5b760fefb35d", + "action": "delete_file", + "applied": false + }, + { + "kind": "unreferenced_pending", + "key": "blobs/3a7b538ee81c67aa0bfd549349a001bbbe727ec8b158dcf8525b0e86e44f2cc8", + "action": "delete_file", + "applied": false + }, + { + "kind": "hash_mismatch", + "key": "blobs/86d4cd5182d5620e915db1e3363c153bd1b619ef09be6a6918fc194e67c92dff", + "action": "report", + "applied": false + }, + { + "kind": "missing_blob", + "key": "blobs/e18d9f81534696cf0ce0143f1c2c4b540873bc7f39e08cbbfe584d6b5212723e", + "action": "mark_failed", + "applied": false + } + ] + }, + { + "phase": "fault", + "mode": "apply", + "target_files": 7, + "go_heap_peak_bytes": 3289520, + "stats": { + "elapsed_ms": 62, + "sql_queries": 41, + "s3_calls": 18, + "scanned_files": 7, + "scanned_objects": 9, + "findings": 6, + "repaired": 5 + }, + "findings": [ + { + "kind": "stale_staging", + "key": "staging/old", + "action": "delete_object", + "applied": true + }, + { + "kind": "orphan_blob", + "key": "blobs/ffea09ba14e194c3863fd5338b4994073275de7f1cfb799a4b82cf27340753b3", + "action": "delete_object", + "applied": true + }, + { + "kind": "unreferenced_ready", + "key": "blobs/0827f495ee64051e55f98e1d89fa78f6e85df2a36c08745dd73b5b760fefb35d", + "action": "delete_file", + "applied": true + }, + { + "kind": "unreferenced_pending", + "key": "blobs/3a7b538ee81c67aa0bfd549349a001bbbe727ec8b158dcf8525b0e86e44f2cc8", + "action": "delete_file", + "applied": true + }, + { + "kind": "hash_mismatch", + "key": "blobs/86d4cd5182d5620e915db1e3363c153bd1b619ef09be6a6918fc194e67c92dff", + "action": "report", + "applied": false + }, + { + "kind": "missing_blob", + "key": "blobs/e18d9f81534696cf0ce0143f1c2c4b540873bc7f39e08cbbfe584d6b5212723e", + "action": "mark_failed", + "applied": true + } + ] + }, + { + "phase": "fault", + "mode": "repeat", + "target_files": 7, + "go_heap_peak_bytes": 2437032, + "stats": { + "elapsed_ms": 37, + "sql_queries": 23, + "s3_calls": 10, + "scanned_files": 5, + "scanned_objects": 5, + "findings": 1, + "repaired": 0 + }, + "findings": [ + { + "kind": "hash_mismatch", + "key": "blobs/86d4cd5182d5620e915db1e3363c153bd1b619ef09be6a6918fc194e67c92dff", + "action": "report", + "applied": false + } + ] + }, + { + "phase": "cost", + "mode": "dry", + "target_files": 100, + "go_heap_peak_bytes": 3875232, + "stats": { + "elapsed_ms": 184, + "sql_queries": 2, + "s3_calls": 103, + "scanned_files": 100, + "scanned_objects": 100, + "findings": 0, + "repaired": 0 + } + }, + { + "phase": "cost", + "mode": "dry", + "target_files": 1000, + "go_heap_peak_bytes": 4159720, + "stats": { + "elapsed_ms": 2160, + "sql_queries": 3, + "s3_calls": 1004, + "scanned_files": 1000, + "scanned_objects": 1000, + "findings": 0, + "repaired": 0 + } + }, + { + "phase": "cost", + "mode": "dry", + "target_files": 5000, + "go_heap_peak_bytes": 5157552, + "stats": { + "elapsed_ms": 10121, + "sql_queries": 11, + "s3_calls": 5012, + "scanned_files": 5000, + "scanned_objects": 5000, + "findings": 0, + "repaired": 0 + } + }, + { + "fault_checks": "passed", + "phase": "complete", + "sizes": [ + 100, + 1000, + 5000 + ] + } +] diff --git a/evidence/upload-memory-initial-512MiB.json b/evidence/upload-memory-initial-512MiB.json new file mode 100644 index 00000000..797ccd7c --- /dev/null +++ b/evidence/upload-memory-initial-512MiB.json @@ -0,0 +1,1447 @@ +{ + "schema_version": 1, + "started_at": "2026-11-16T08:52:10.242029+00:00", + "ok": true, + "driver": { + "os": "Windows", + "architecture": "AMD64", + "logical_cpus": 16, + "python": "3.13.1" + }, + "measurement": { + "source": "docker stats JSON stream", + "cadence": "Docker-managed, typically about 1 second", + "limitation": "sampled peaks may miss sub-second maxima; includes post-upload sampling tail", + "memory_semantics": "Docker CLI memory usage (working set, cache excluded where supported)", + "baseline_seconds": 3, + "tail_seconds": 3, + "same_bytes_per_case": true, + "p95_method": "nearest rank", + "png_compression": "zlib level 1; random RGB; npAD padding to exact byte limit" + }, + "cases": [ + { + "name": "small", + "width": 512, + "height": 512, + "seed": 41, + "payload_bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "requests": 10, + "attempted_bytes": 10485760, + "successful_bytes": 10485760, + "accepted": 10, + "status_counts": { + "201": 10 + }, + "errors": 0, + "duration_s": 0.24706, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 209.3365, + "p95_ms": 242.959, + "max_ms": 242.959 + }, + "memory": { + "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504": { + "baseline_bytes": 70988595, + "sampled_peak_bytes": 72519516, + "delta_bytes": 1530921, + "baseline_samples": 14, + "load_samples": 12 + }, + "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9": { + "baseline_bytes": 73526149, + "sampled_peak_bytes": 74606182, + "delta_bytes": 1080033, + "baseline_samples": 14, + "load_samples": 12 + } + }, + "memory_samples": [ + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70611108, + "limit_bytes": 536870912, + "elapsed_s": 1.554355, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73767322, + "limit_bytes": 536870912, + "elapsed_s": 1.554401, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70611108, + "limit_bytes": 536870912, + "elapsed_s": 1.554415, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73767322, + "limit_bytes": 536870912, + "elapsed_s": 1.554424, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70611108, + "limit_bytes": 536870912, + "elapsed_s": 2.053812, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73767322, + "limit_bytes": 536870912, + "elapsed_s": 2.053838, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70611108, + "limit_bytes": 536870912, + "elapsed_s": 2.053849, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73767322, + "limit_bytes": 536870912, + "elapsed_s": 2.053858, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71523369, + "limit_bytes": 536870912, + "elapsed_s": 2.553902, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73410806, + "limit_bytes": 536870912, + "elapsed_s": 2.553997, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71523369, + "limit_bytes": 536870912, + "elapsed_s": 2.554068, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73410806, + "limit_bytes": 536870912, + "elapsed_s": 2.554102, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71523369, + "limit_bytes": 536870912, + "elapsed_s": 3.054128, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73410806, + "limit_bytes": 536870912, + "elapsed_s": 3.054147, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71523369, + "limit_bytes": 536870912, + "elapsed_s": 3.054156, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73410806, + "limit_bytes": 536870912, + "elapsed_s": 3.054163, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70904709, + "limit_bytes": 536870912, + "elapsed_s": 3.553887, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73400320, + "limit_bytes": 536870912, + "elapsed_s": 3.553907, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70904709, + "limit_bytes": 536870912, + "elapsed_s": 3.553916, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73400320, + "limit_bytes": 536870912, + "elapsed_s": 3.553923, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70904709, + "limit_bytes": 536870912, + "elapsed_s": 4.054015, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73400320, + "limit_bytes": 536870912, + "elapsed_s": 4.054035, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70904709, + "limit_bytes": 536870912, + "elapsed_s": 4.054044, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73400320, + "limit_bytes": 536870912, + "elapsed_s": 4.054051, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70988595, + "limit_bytes": 536870912, + "elapsed_s": 4.554044, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73526149, + "limit_bytes": 536870912, + "elapsed_s": 4.554062, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70988595, + "limit_bytes": 536870912, + "elapsed_s": 4.55407, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73526149, + "limit_bytes": 536870912, + "elapsed_s": 4.554077, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70988595, + "limit_bytes": 536870912, + "elapsed_s": 5.054131, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73526149, + "limit_bytes": 536870912, + "elapsed_s": 5.054155, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 70988595, + "limit_bytes": 536870912, + "elapsed_s": 5.054167, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 73526149, + "limit_bytes": 536870912, + "elapsed_s": 5.054177, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 5.553611, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74050437, + "limit_bytes": 536870912, + "elapsed_s": 5.553634, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 5.553645, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74050437, + "limit_bytes": 536870912, + "elapsed_s": 5.553654, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 6.054001, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74050437, + "limit_bytes": 536870912, + "elapsed_s": 6.054019, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 6.054027, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74050437, + "limit_bytes": 536870912, + "elapsed_s": 6.054035, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 6.553987, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74606182, + "limit_bytes": 536870912, + "elapsed_s": 6.554075, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 6.554098, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74606182, + "limit_bytes": 536870912, + "elapsed_s": 6.55411, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 7.054042, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74606182, + "limit_bytes": 536870912, + "elapsed_s": 7.054066, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 7.05408, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74606182, + "limit_bytes": 536870912, + "elapsed_s": 7.054089, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 72519516, + "limit_bytes": 536870912, + "elapsed_s": 7.554107, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74081894, + "limit_bytes": 536870912, + "elapsed_s": 7.55413, + "case": "small", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 72519516, + "limit_bytes": 536870912, + "elapsed_s": 7.55414, + "case": "small", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74081894, + "limit_bytes": 536870912, + "elapsed_s": 7.554149, + "case": "small", + "phase": "load" + } + ], + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-98e1-7260-b0a1-a77c2ab5d3dc", + "latency_ms": 234.22, + "start_offset_ms": 2.977 + }, + { + "index": 1, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-98cc-74e7-bea2-6b4b7f4ae8c1", + "latency_ms": 182.4, + "start_offset_ms": 0.395 + }, + { + "index": 2, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-98cf-7ad9-8fd2-5350ec5e18cb", + "latency_ms": 187.238, + "start_offset_ms": 0.847 + }, + { + "index": 3, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-98dc-7e44-99a2-f23f64c2b07e", + "latency_ms": 213.514, + "start_offset_ms": 2.404 + }, + { + "index": 4, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-98cb-7e86-9576-d7e237205de2", + "latency_ms": 228.836, + "start_offset_ms": 0.113 + }, + { + "index": 5, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-98d0-7a4a-8619-9ea12d62ffa4", + "latency_ms": 196.465, + "start_offset_ms": 1.711 + }, + { + "index": 6, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-98d0-703a-a896-e60b46f8575b", + "latency_ms": 190.544, + "start_offset_ms": 1.582 + }, + { + "index": 7, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-98e8-73eb-9314-b0d1e4694aa4", + "latency_ms": 217.92, + "start_offset_ms": 2.203 + }, + { + "index": 8, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-98d6-7f20-8813-3a50e2d9d06f", + "latency_ms": 205.159, + "start_offset_ms": 2.551 + }, + { + "index": 9, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-98e4-7729-aa83-a29798927c1b", + "latency_ms": 242.959, + "start_offset_ms": 2.811 + } + ], + "download": { + "ok": true, + "bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "status": 200 + }, + "ok": true + }, + { + "name": "limit", + "width": 1600, + "height": 1600, + "seed": 42, + "payload_bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "requests": 10, + "attempted_bytes": 104857600, + "successful_bytes": 104857600, + "accepted": 10, + "status_counts": { + "201": 10 + }, + "errors": 0, + "duration_s": 2.709547, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 2680.373, + "p95_ms": 2705.536, + "max_ms": 2705.536 + }, + "memory": { + "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504": { + "baseline_bytes": 75948360, + "sampled_peak_bytes": 75948360, + "delta_bytes": 0, + "baseline_samples": 14, + "load_samples": 22 + }, + "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9": { + "baseline_bytes": 74165780, + "sampled_peak_bytes": 74207724, + "delta_bytes": 41944, + "baseline_samples": 14, + "load_samples": 22 + } + }, + "memory_samples": [ + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71827456, + "limit_bytes": 536870912, + "elapsed_s": 9.553749, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74186752, + "limit_bytes": 536870912, + "elapsed_s": 9.55378, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71827456, + "limit_bytes": 536870912, + "elapsed_s": 9.553791, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74186752, + "limit_bytes": 536870912, + "elapsed_s": 9.553799, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71827456, + "limit_bytes": 536870912, + "elapsed_s": 10.054538, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74186752, + "limit_bytes": 536870912, + "elapsed_s": 10.054562, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71827456, + "limit_bytes": 536870912, + "elapsed_s": 10.054573, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74186752, + "limit_bytes": 536870912, + "elapsed_s": 10.054585, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 10.553817, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74060923, + "limit_bytes": 536870912, + "elapsed_s": 10.553838, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 10.553846, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74060923, + "limit_bytes": 536870912, + "elapsed_s": 10.553853, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 11.054355, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74060923, + "limit_bytes": 536870912, + "elapsed_s": 11.054382, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71754056, + "limit_bytes": 536870912, + "elapsed_s": 11.054393, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74060923, + "limit_bytes": 536870912, + "elapsed_s": 11.054402, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71879885, + "limit_bytes": 536870912, + "elapsed_s": 11.554074, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74595697, + "limit_bytes": 536870912, + "elapsed_s": 11.554094, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71879885, + "limit_bytes": 536870912, + "elapsed_s": 11.554102, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74595697, + "limit_bytes": 536870912, + "elapsed_s": 11.554109, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71879885, + "limit_bytes": 536870912, + "elapsed_s": 12.053703, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74595697, + "limit_bytes": 536870912, + "elapsed_s": 12.053721, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71879885, + "limit_bytes": 536870912, + "elapsed_s": 12.053731, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74595697, + "limit_bytes": 536870912, + "elapsed_s": 12.053738, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 75948360, + "limit_bytes": 536870912, + "elapsed_s": 12.553922, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74165780, + "limit_bytes": 536870912, + "elapsed_s": 12.553949, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 75948360, + "limit_bytes": 536870912, + "elapsed_s": 12.553972, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74165780, + "limit_bytes": 536870912, + "elapsed_s": 12.553981, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 75948360, + "limit_bytes": 536870912, + "elapsed_s": 13.053641, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74165780, + "limit_bytes": 536870912, + "elapsed_s": 13.053685, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 75948360, + "limit_bytes": 536870912, + "elapsed_s": 13.053705, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74165780, + "limit_bytes": 536870912, + "elapsed_s": 13.053722, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71775027, + "limit_bytes": 536870912, + "elapsed_s": 13.554034, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74207724, + "limit_bytes": 536870912, + "elapsed_s": 13.554109, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71775027, + "limit_bytes": 536870912, + "elapsed_s": 13.554155, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74207724, + "limit_bytes": 536870912, + "elapsed_s": 13.554199, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71775027, + "limit_bytes": 536870912, + "elapsed_s": 14.053787, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74207724, + "limit_bytes": 536870912, + "elapsed_s": 14.053818, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71775027, + "limit_bytes": 536870912, + "elapsed_s": 14.053834, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 74207724, + "limit_bytes": 536870912, + "elapsed_s": 14.053846, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71785513, + "limit_bytes": 536870912, + "elapsed_s": 14.553747, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 59276001, + "limit_bytes": 536870912, + "elapsed_s": 14.553787, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71785513, + "limit_bytes": 536870912, + "elapsed_s": 14.553807, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 59276001, + "limit_bytes": 536870912, + "elapsed_s": 14.553822, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71785513, + "limit_bytes": 536870912, + "elapsed_s": 15.053972, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 59276001, + "limit_bytes": 536870912, + "elapsed_s": 15.054003, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 71785513, + "limit_bytes": 536870912, + "elapsed_s": 15.054018, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 59276001, + "limit_bytes": 536870912, + "elapsed_s": 15.05403, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42834330, + "limit_bytes": 536870912, + "elapsed_s": 15.553914, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46252687, + "limit_bytes": 536870912, + "elapsed_s": 15.553937, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42834330, + "limit_bytes": 536870912, + "elapsed_s": 15.553948, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46252687, + "limit_bytes": 536870912, + "elapsed_s": 15.553957, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42834330, + "limit_bytes": 536870912, + "elapsed_s": 16.054132, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46252687, + "limit_bytes": 536870912, + "elapsed_s": 16.054156, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42834330, + "limit_bytes": 536870912, + "elapsed_s": 16.054168, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46252687, + "limit_bytes": 536870912, + "elapsed_s": 16.054178, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 16.553867, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46200259, + "limit_bytes": 536870912, + "elapsed_s": 16.55389, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 16.553902, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46200259, + "limit_bytes": 536870912, + "elapsed_s": 16.553912, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 17.054159, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46200259, + "limit_bytes": 536870912, + "elapsed_s": 17.054188, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 17.054201, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 46200259, + "limit_bytes": 536870912, + "elapsed_s": 17.054213, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 17.553554, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 45990543, + "limit_bytes": 536870912, + "elapsed_s": 17.553581, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 17.553593, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 45990543, + "limit_bytes": 536870912, + "elapsed_s": 17.553603, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 18.053727, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 45990543, + "limit_bytes": 536870912, + "elapsed_s": 18.053788, + "case": "limit", + "phase": "load" + }, + { + "container_id": "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "name": "lab4-api-1", + "used_bytes": 42016440, + "limit_bytes": 536870912, + "elapsed_s": 18.053818, + "case": "limit", + "phase": "load" + }, + { + "container_id": "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9", + "name": "lab4-api-2", + "used_bytes": 45990543, + "limit_bytes": 536870912, + "elapsed_s": 18.053843, + "case": "limit", + "phase": "load" + } + ], + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-b839-7b2f-984a-93a2883c43eb", + "latency_ms": 2702.428, + "start_offset_ms": 1.592 + }, + { + "index": 1, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-b82b-75a2-b9b4-8c3a995484b5", + "latency_ms": 2686.356, + "start_offset_ms": 3.615 + }, + { + "index": 2, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-b823-71c0-bb4d-878c86b4d327", + "latency_ms": 2660.235, + "start_offset_ms": 0.054 + }, + { + "index": 3, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-b827-7e3f-b28a-45ee18b048b3", + "latency_ms": 2674.457, + "start_offset_ms": 2.073 + }, + { + "index": 4, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-b824-7893-9e31-4daa314103ff", + "latency_ms": 2664.117, + "start_offset_ms": 2.367 + }, + { + "index": 5, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-b825-7d79-9db9-dcdd32fc2956", + "latency_ms": 2669.858, + "start_offset_ms": 2.53 + }, + { + "index": 6, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-b83a-721b-a954-4e3e8dcdd85e", + "latency_ms": 2692.123, + "start_offset_ms": 2.662 + }, + { + "index": 7, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8d749be14a20bdd2", + "attachment_id": "01a1e38e-b828-73a4-8eb6-1b93f5d1ea71", + "latency_ms": 2678.474, + "start_offset_ms": 2.901 + }, + { + "index": 8, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-b83d-74ae-9f0f-84d02a39ec10", + "latency_ms": 2705.536, + "start_offset_ms": 3.178 + }, + { + "index": 9, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "0bbd6ea08d8cf283", + "attachment_id": "01a1e38e-b829-768c-a844-e66b4dabdd64", + "latency_ms": 2682.272, + "start_offset_ms": 3.399 + } + ], + "download": { + "ok": true, + "bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "status": 200 + }, + "ok": true + } + ], + "api_container_ids": [ + "f51ea35133e372283602783fb843b670493b0c85b167b940d2147d27a2a20504", + "29b0504314716e4a8afe5f787e08c50071074be492ee10cbf802d7344f5e94f9" + ], + "stage": "complete" +} diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..ab47c9f6 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,1486 @@ +{ + "schema_version": 1, + "started_at": "2026-11-16T09:16:00.126816+00:00", + "ok": true, + "driver": { + "os": "Windows", + "architecture": "AMD64", + "logical_cpus": 16, + "python": "3.13.1" + }, + "measurement": { + "source": "docker stats JSON stream", + "cadence": "Docker-managed, typically about 1 second", + "limitation": "sampled peaks may miss sub-second maxima; includes post-upload sampling tail", + "memory_semantics": "Docker CLI memory usage (working set, cache excluded where supported)", + "baseline_seconds": 3, + "tail_seconds": 3, + "same_bytes_per_case": true, + "p95_method": "nearest rank", + "profile": "standard", + "png_compression": "zlib level 1; random RGB; npAD padding to exact byte limit" + }, + "cases": [ + { + "name": "small", + "width": 512, + "height": 512, + "seed": 41, + "payload_bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "bit_depth": 8, + "color_type": 2, + "nominal_decoded_bytes": 1048576, + "requests": 10, + "attempted_bytes": 10485760, + "successful_bytes": 10485760, + "accepted": 10, + "status_counts": { + "201": 10 + }, + "errors": 0, + "duration_s": 0.264958, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 216.3425, + "p95_ms": 259.146, + "max_ms": 259.146 + }, + "memory": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "baseline_bytes": 47123005, + "sampled_peak_bytes": 48685384, + "delta_bytes": 1562379, + "baseline_samples": 14, + "load_samples": 12 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "baseline_bytes": 70495764, + "sampled_peak_bytes": 72980890, + "delta_bytes": 2485126, + "baseline_samples": 14, + "load_samples": 12 + } + }, + "memory_samples": [ + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 1.552227, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70432850, + "limit_bytes": 2147483648, + "elapsed_s": 1.552254, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 1.552277, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70432850, + "limit_bytes": 2147483648, + "elapsed_s": 1.55229, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 2.052078, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70432850, + "limit_bytes": 2147483648, + "elapsed_s": 2.052098, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 2.052107, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70432850, + "limit_bytes": 2147483648, + "elapsed_s": 2.052114, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 2.552466, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 2.552484, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 2.552494, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 2.552502, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 3.05489, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 3.054955, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 3.054973, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 3.054985, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47259320, + "limit_bytes": 2147483648, + "elapsed_s": 3.552275, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 3.552294, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47259320, + "limit_bytes": 2147483648, + "elapsed_s": 3.552303, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 3.552311, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47259320, + "limit_bytes": 2147483648, + "elapsed_s": 4.052714, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 4.052736, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47259320, + "limit_bytes": 2147483648, + "elapsed_s": 4.052748, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70464307, + "limit_bytes": 2147483648, + "elapsed_s": 4.052758, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 4.552224, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70495764, + "limit_bytes": 2147483648, + "elapsed_s": 4.552243, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 4.552252, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70495764, + "limit_bytes": 2147483648, + "elapsed_s": 4.552259, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 5.052111, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70495764, + "limit_bytes": 2147483648, + "elapsed_s": 5.052133, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 47123005, + "limit_bytes": 2147483648, + "elapsed_s": 5.052145, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 70495764, + "limit_bytes": 2147483648, + "elapsed_s": 5.052155, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48538583, + "limit_bytes": 2147483648, + "elapsed_s": 5.552368, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 5.552386, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48538583, + "limit_bytes": 2147483648, + "elapsed_s": 5.552396, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 5.552405, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48538583, + "limit_bytes": 2147483648, + "elapsed_s": 6.052735, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 6.052754, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48538583, + "limit_bytes": 2147483648, + "elapsed_s": 6.052763, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 6.05277, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 6.552578, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 6.552597, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 6.552606, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 6.552614, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 7.052501, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 7.052525, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 7.052537, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72844575, + "limit_bytes": 2147483648, + "elapsed_s": 7.052547, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48685384, + "limit_bytes": 2147483648, + "elapsed_s": 7.552404, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72980890, + "limit_bytes": 2147483648, + "elapsed_s": 7.552437, + "case": "small", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48685384, + "limit_bytes": 2147483648, + "elapsed_s": 7.552456, + "case": "small", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72980890, + "limit_bytes": 2147483648, + "elapsed_s": 7.552471, + "case": "small", + "phase": "load" + } + ], + "memory_error": null, + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-6951-7167-abfa-a38491b35373", + "latency_ms": 188.401, + "start_offset_ms": 1.165 + }, + { + "index": 1, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-6963-76eb-930a-7301153b4b2e", + "latency_ms": 259.146, + "start_offset_ms": 4.552 + }, + { + "index": 2, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-694e-7a1b-94a5-4bfdc97a6b89", + "latency_ms": 222.994, + "start_offset_ms": 0.057 + }, + { + "index": 3, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-6953-7c75-8573-aa968cc3a9fb", + "latency_ms": 204.592, + "start_offset_ms": 3.426 + }, + { + "index": 4, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-695f-7600-9615-e342a127a289", + "latency_ms": 240.948, + "start_offset_ms": 2.298 + }, + { + "index": 5, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-6963-77a6-a3ac-51ff5db2b1d0", + "latency_ms": 229.677, + "start_offset_ms": 3.597 + }, + { + "index": 6, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-695d-7605-a2fc-b80becde2a20", + "latency_ms": 249.598, + "start_offset_ms": 5.312 + }, + { + "index": 7, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-6951-7c8d-8b43-4693babc1835", + "latency_ms": 195.633, + "start_offset_ms": 1.403 + }, + { + "index": 8, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-6953-72df-8b4a-9bf09a2eaf90", + "latency_ms": 191.092, + "start_offset_ms": 1.651 + }, + { + "index": 9, + "status": 201, + "ok": true, + "request_bytes": 1048576, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-6953-79b0-93e5-b1257e6dbec1", + "latency_ms": 209.691, + "start_offset_ms": 4.065 + } + ], + "download": { + "ok": true, + "bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "status": 200 + }, + "state_before": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "state_after": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "ok": true + }, + { + "name": "limit", + "width": 1600, + "height": 1600, + "seed": 42, + "payload_bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "bit_depth": 8, + "color_type": 2, + "nominal_decoded_bytes": 10240000, + "requests": 10, + "attempted_bytes": 104857600, + "successful_bytes": 104857600, + "accepted": 10, + "status_counts": { + "201": 10 + }, + "errors": 0, + "duration_s": 2.002592, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 1865.3935000000001, + "p95_ms": 1997.061, + "max_ms": 1997.061 + }, + "memory": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "baseline_bytes": 48559555, + "sampled_peak_bytes": 57168364, + "delta_bytes": 8608809, + "baseline_samples": 14, + "load_samples": 20 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "baseline_bytes": 72865546, + "sampled_peak_bytes": 72928461, + "delta_bytes": 62915, + "baseline_samples": 14, + "load_samples": 20 + } + }, + "memory_samples": [ + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 10.573123, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 10.573149, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 10.573159, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 10.573167, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 11.05276, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 11.052779, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48549069, + "limit_bytes": 2147483648, + "elapsed_s": 11.052788, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 11.052796, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48674898, + "limit_bytes": 2147483648, + "elapsed_s": 11.552316, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 11.552335, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48674898, + "limit_bytes": 2147483648, + "elapsed_s": 11.552344, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 11.552352, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48674898, + "limit_bytes": 2147483648, + "elapsed_s": 12.052146, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 12.052166, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48674898, + "limit_bytes": 2147483648, + "elapsed_s": 12.052175, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 12.052183, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 12.552653, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 12.552675, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 12.552684, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 12.552691, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 13.052429, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 13.052448, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 13.052457, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72991375, + "limit_bytes": 2147483648, + "elapsed_s": 13.052464, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 13.552345, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 13.552364, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 13.552374, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 13.552381, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 14.052742, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 14.052794, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48559555, + "limit_bytes": 2147483648, + "elapsed_s": 14.052818, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72865546, + "limit_bytes": 2147483648, + "elapsed_s": 14.052836, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48632955, + "limit_bytes": 2147483648, + "elapsed_s": 14.552433, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72928461, + "limit_bytes": 2147483648, + "elapsed_s": 14.552491, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48632955, + "limit_bytes": 2147483648, + "elapsed_s": 14.552531, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72928461, + "limit_bytes": 2147483648, + "elapsed_s": 14.552568, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48632955, + "limit_bytes": 2147483648, + "elapsed_s": 15.052459, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72928461, + "limit_bytes": 2147483648, + "elapsed_s": 15.052483, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 48632955, + "limit_bytes": 2147483648, + "elapsed_s": 15.052493, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 72928461, + "limit_bytes": 2147483648, + "elapsed_s": 15.052503, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57168364, + "limit_bytes": 2147483648, + "elapsed_s": 15.552227, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 66249032, + "limit_bytes": 2147483648, + "elapsed_s": 15.552247, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57168364, + "limit_bytes": 2147483648, + "elapsed_s": 15.552258, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 66249032, + "limit_bytes": 2147483648, + "elapsed_s": 15.552268, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57168364, + "limit_bytes": 2147483648, + "elapsed_s": 16.052494, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 66249032, + "limit_bytes": 2147483648, + "elapsed_s": 16.052523, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 57168364, + "limit_bytes": 2147483648, + "elapsed_s": 16.052534, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 66249032, + "limit_bytes": 2147483648, + "elapsed_s": 16.052543, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44260393, + "limit_bytes": 2147483648, + "elapsed_s": 16.552499, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44574966, + "limit_bytes": 2147483648, + "elapsed_s": 16.55252, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44260393, + "limit_bytes": 2147483648, + "elapsed_s": 16.552528, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44574966, + "limit_bytes": 2147483648, + "elapsed_s": 16.552536, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44260393, + "limit_bytes": 2147483648, + "elapsed_s": 17.052156, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44574966, + "limit_bytes": 2147483648, + "elapsed_s": 17.052175, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44260393, + "limit_bytes": 2147483648, + "elapsed_s": 17.052184, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44574966, + "limit_bytes": 2147483648, + "elapsed_s": 17.052191, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44029706, + "limit_bytes": 2147483648, + "elapsed_s": 17.552413, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44700795, + "limit_bytes": 2147483648, + "elapsed_s": 17.552431, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44029706, + "limit_bytes": 2147483648, + "elapsed_s": 17.55244, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44700795, + "limit_bytes": 2147483648, + "elapsed_s": 17.552448, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44029706, + "limit_bytes": 2147483648, + "elapsed_s": 18.052194, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44700795, + "limit_bytes": 2147483648, + "elapsed_s": 18.052214, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44029706, + "limit_bytes": 2147483648, + "elapsed_s": 18.052223, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44700795, + "limit_bytes": 2147483648, + "elapsed_s": 18.05223, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44040192, + "limit_bytes": 2147483648, + "elapsed_s": 18.552282, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44606423, + "limit_bytes": 2147483648, + "elapsed_s": 18.552301, + "case": "limit", + "phase": "load" + }, + { + "container_id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "name": "lab4-api-1", + "used_bytes": 44040192, + "limit_bytes": 2147483648, + "elapsed_s": 18.55231, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "name": "lab4-api-2", + "used_bytes": 44606423, + "limit_bytes": 2147483648, + "elapsed_s": 18.552318, + "case": "limit", + "phase": "load" + } + ], + "memory_error": null, + "results": [ + { + "index": 0, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-8ca6-798c-8e8b-1ed53bd5a88c", + "latency_ms": 1775.107, + "start_offset_ms": 1.063 + }, + { + "index": 1, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-8cb9-7156-855d-d6b3a09784a4", + "latency_ms": 1997.061, + "start_offset_ms": 4.444 + }, + { + "index": 2, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-8ca4-750d-bb86-8f70258ed58e", + "latency_ms": 1772.039, + "start_offset_ms": 0.049 + }, + { + "index": 3, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-8ca9-7584-b1e2-ada08b5b1205", + "latency_ms": 1971.765, + "start_offset_ms": 2.492 + }, + { + "index": 4, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-8cb9-7129-90e1-7da4ca737150", + "latency_ms": 1992.107, + "start_offset_ms": 1.967 + }, + { + "index": 5, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-8ca7-739d-a2ea-084ff5a7510f", + "latency_ms": 1773.813, + "start_offset_ms": 1.303 + }, + { + "index": 6, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-8ca8-7142-90e9-1f1d5808c761", + "latency_ms": 1774.874, + "start_offset_ms": 1.831 + }, + { + "index": 7, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-8ca8-7984-a070-31877450bf77", + "latency_ms": 1772.359, + "start_offset_ms": 2.352 + }, + { + "index": 8, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "76f24bb2ebbc2a92", + "attachment_id": "01a1e3a4-8caa-77fc-a3f2-f192cd84cca6", + "latency_ms": 1961.16, + "start_offset_ms": 2.971 + }, + { + "index": 9, + "status": 201, + "ok": true, + "request_bytes": 10485760, + "instance_id": "acce7ec5a5aab35b", + "attachment_id": "01a1e3a4-8ca9-75c2-a410-02e8327f964e", + "latency_ms": 1955.68, + "start_offset_ms": 3.23 + } + ], + "download": { + "ok": true, + "bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "status": 200 + }, + "state_before": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "state_after": { + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918": { + "id": "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd": { + "id": "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "ok": true + } + ], + "api_container_ids": [ + "e0755982c2d34c1849f3c6aed91fc3c6fb659d9114f6cab52ad1908fafffa918", + "4705eb662d229d90626175762a8d73a46410d642a14f75aa80285442335032bd" + ], + "stage": "complete" +} -- 2.54.0 From 80c4a56200e3eb2389e9338fb9bb7e5b7bc63707 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 18 Nov 2026 11:43:27 +0300 Subject: [PATCH 6/6] =?UTF-8?q?=D0=94=D0=BE=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=B0=D0=B9=D1=82=D0=B5=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D1=8E=20=D0=B8=20=D1=81=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D1=83=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9?= =?UTF-8?q?=20=D0=BF=D0=BE=20=D0=B8=D1=82=D0=BE=D0=B3=D0=B0=D0=BC=20=D1=80?= =?UTF-8?q?=D0=B5=D0=B2=D1=8C=D1=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 4 +- cmd/reconcile/main.go | 9 +- cmd/reconcile/main_test.go | 35 +++++ cmd/s3init/main.go | 7 +- cmd/server/main.go | 55 ++++--- compose.yaml | 3 + internal/app/app.go | 30 ++-- internal/app/attachment_test.go | 72 ++++++--- internal/attachment/attachment.go | 24 ++- internal/attachment/blob.go | 12 +- internal/attachment/handler.go | 64 +++++--- internal/attachment/reconcile.go | 147 ++++++++++-------- internal/attachment/reconcile_test.go | 37 ++++- internal/attachment/repository.go | 23 +++ internal/attachment/service.go | 124 +++++++++++---- internal/attachment/service_test.go | 22 ++- internal/attachment/store/postgres.go | 86 +++++----- internal/attachment/validation.go | 7 +- internal/attachment/validation_test.go | 56 ++++++- internal/chat/store/postgres.go | 5 +- internal/platform/config/config.go | 8 +- internal/platform/s3store/bootstrap_test.go | 5 +- internal/platform/s3store/integration_test.go | 5 +- internal/platform/s3store/store.go | 19 ++- internal/platform/s3store/store_test.go | 11 +- internal/testenv/s3.go | 7 +- tools/reconcilebench/main.go | 3 +- 27 files changed, 605 insertions(+), 275 deletions(-) create mode 100644 cmd/reconcile/main_test.go create mode 100644 internal/attachment/repository.go diff --git a/REPORT.md b/REPORT.md index 77fe5799..90eb7880 100644 --- a/REPORT.md +++ b/REPORT.md @@ -78,7 +78,7 @@ Grace period — 1 h, минимум 2 min, выше бюджета upload. Св | Используемый ready без объекта | Пометить файл failed, сохранить вложения, сообщения и связи. | | Неверный размер или SHA-256 (`--deep`) | Только сообщить, не удалять используемые данные. | -Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает файл; простое появление объекта само не меняет failed на ready. +Непривязанное к сообщению логическое вложение не считается мусором — его можно прикрепить позже. Повторный apply безопасен; уже обработанный missing не создаёт нового исправления. Повторная проверенная загрузка восстанавливает отсутствующий blob и файл в failed. Для ready с существующим объектом того же размера действует быстрый Stat-путь. Deep-повреждение такого ready диагностируется и требует отдельного восстановления из достоверных байтов. Простое появление объекта само не меняет failed на ready. ```sh make reconcile @@ -136,7 +136,7 @@ Bootstrap — одноразовый `s3-init` с отдельным admin-кл | --- | ---: | ---: | --- | | Progressive JPEG RGB 4:4:4, 8000×5000 | 469 711 | 585.2 | 201, checksum совпал, без OOM/restart | | Progressive JPEG CMYK 4:4:4, 8000×5000 | 939 079 | 945.5 | 201, checksum совпал, без OOM/restart | -| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.3 | 201, checksum совпал, без OOM/restart | +| PNG RGBA64, 40 000 000×1 | 1 397 354 | 1 290.2 | 201, checksum совпал, без OOM/restart | Полные сэмплы, container limits и состояния до/после — `evidence/decoder-memory-*.json`. Старый прогон при 512 MiB оставлен отдельно как историческое сравнение, не как доказательство текущего лимита. В стресс-серии процесс уже был прогрет предыдущими изображениями: baseline может включать не возвращённую ОС память. Три выбранных формата не доказывают абсолютный максимум для всех допустимых файлов и сочетаний нагрузки. diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go index b7090a83..97b79811 100644 --- a/cmd/reconcile/main.go +++ b/cmd/reconcile/main.go @@ -43,7 +43,7 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m") return 2 } - dbConfig, err := config.LoadMigrate(os.LookupEnv) + databaseURL, err := config.LoadDatabaseURL(os.LookupEnv) if err != nil { fmt.Fprintln(errOut, "configuration error:", err) return 2 @@ -54,12 +54,12 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { fmt.Fprintln(errOut, "configuration error:", err) return 2 } - blobs, err := s3store.New(s3Config.Endpoint, s3Config.Region, s3Config.Bucket, s3Config.AccessKey, s3Config.SecretKey) + blobs, err := s3store.New(s3Config) if err != nil { fmt.Fprintln(errOut, "configuration error:", err) return 2 } - db, err := postgres.NewPool(ctx, dbConfig.DatabaseURL, 10) + db, err := postgres.NewPool(ctx, databaseURL, 10) if err != nil { fmt.Fprintln(errOut, "reconcile:", err) return 1 @@ -76,7 +76,8 @@ func run(ctx context.Context, args []string, out, errOut io.Writer) int { defer cancel() encoder := json.NewEncoder(out) var outputErr error - stats, runErr := r.Run(scanCtx, *apply, *deep, func(f attachment.Finding) { + options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep} + stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) { if outputErr == nil { outputErr = encoder.Encode(f) if outputErr != nil { diff --git a/cmd/reconcile/main_test.go b/cmd/reconcile/main_test.go new file mode 100644 index 00000000..806f2d67 --- /dev/null +++ b/cmd/reconcile/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) { + for key, value := range map[string]string{ + "DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable", + "S3_ACCESS_KEY_ID_FILE": "test-access", + "S3_SECRET_ACCESS_KEY_FILE": "test-secret", + } { + path := filepath.Join(t.TempDir(), key) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + t.Setenv(key, path) + } + t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333") + t.Setenv("S3_REGION", "us-east-1") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("APP_LOG_LEVEL", "not-a-log-level") + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Stop before dialing: only configuration relevant to this CLI is checked. + var out, errOut bytes.Buffer + code := run(ctx, nil, &out, &errOut) + if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") { + t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String()) + } +} diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go index db308002..164c3b01 100644 --- a/cmd/s3init/main.go +++ b/cmd/s3init/main.go @@ -3,10 +3,11 @@ package main import ( "context" "fmt" - "messenger/internal/platform/config" - "messenger/internal/platform/s3store" "os" "time" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" ) func main() { @@ -20,7 +21,7 @@ func run() error { if err != nil { return err } - s, err := s3store.New(cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.SecretKey) + s, err := s3store.New(cfg) if err != nil { return err } diff --git a/cmd/server/main.go b/cmd/server/main.go index cb4aec4a..4a3c85b8 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,4 +1,4 @@ -// Command server starts the messenger API over PostgreSQL and Valkey. +// Command server starts the messenger API over PostgreSQL, Valkey and S3. package main import ( @@ -61,25 +61,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } defer sessions.Close() - var blobs attachment.Blobs - var reconciler *attachment.Reconciler - if cfg.S3.Endpoint != "" { - blobs, err = s3store.New(cfg.S3.Endpoint, cfg.S3.Region, cfg.S3.Bucket, cfg.S3.AccessKey, cfg.S3.SecretKey) - if err != nil { - return err - } - reconCfg, e := config.LoadS3(config.ReconcileS3(os.LookupEnv)) - if e != nil { - return e - } - reconBlobs, e := s3store.New(reconCfg.Endpoint, reconCfg.Region, reconCfg.Bucket, reconCfg.AccessKey, reconCfg.SecretKey) - if e != nil { - return e - } - reconciler, err = attachment.NewReconciler(pool, reconBlobs, time.Hour) - if err != nil { - return err - } + blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv) + if err != nil { + return err } ln, err := net.Listen("tcp", cfg.Addr()) @@ -100,8 +84,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, Blobs: blobs, - MaxUploadBytes: cfg.S3.MaxBytes, - MaxImagePixels: cfg.S3.MaxPixels, + AttachmentsCfg: attachment.Config{ + MaxBytes: cfg.S3.MaxBytes, + MaxPixels: cfg.S3.MaxPixels, + }, }) var debugLn net.Listener if cfg.DebugAddr != "" { @@ -124,7 +110,7 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return case <-ticker.C: scan, cancel := context.WithTimeout(ctx, 2*time.Minute) - stats, err := reconciler.Run(scan, true, false, nil) + stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil) cancel() if err != nil { log.Warn("attachment cleanup incomplete", "error", err) @@ -149,6 +135,29 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) return err } +func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) { + if cfg.Endpoint == "" { + return nil, nil, nil + } + blobs, err := s3store.New(cfg) + if err != nil { + return nil, nil, err + } + reconCfg, err := config.LoadS3(config.ReconcileS3(lookup)) + if err != nil { + return nil, nil, err + } + reconBlobs, err := s3store.New(reconCfg) + if err != nil { + return nil, nil, err + } + reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour) + if err != nil { + return nil, nil, err + } + return blobs, reconciler, nil +} + // publishDebugVars adds to expvar what the load tests read from a replica. func publishDebugVars(alias string, pool *pgxpool.Pool) { expvar.NewString("instance").Set(alias) diff --git a/compose.yaml b/compose.yaml index bc08ab33..6290c342 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,9 @@ name: lab4 services: lb: image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa + depends_on: + api: + condition: service_healthy volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro ports: diff --git a/internal/app/app.go b/internal/app/app.go index 3949add5..9ab9a811 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -39,20 +39,23 @@ type Deps struct { Sessions *redis.Client SessionTTL time.Duration MaxBodyBytes int64 - // RequestTimeout bounds every API request; zero means the default. - RequestTimeout time.Duration - InstanceID string + // Zero timeouts use the defaults below. + RequestTimeout time.Duration + BinaryRequestTimeout time.Duration + InstanceID string // IdempotencyTTL zero means the contract's 24 hours. IdempotencyTTL time.Duration Blobs attachment.Blobs - MaxUploadBytes int64 - MaxImagePixels int64 + AttachmentsCfg attachment.Config } -// defaultRequestTimeout stays below the server's 10s write timeout, so a +// defaultRequestTimeout stays below the server's write timeout, so a // request that gives up on a dependency can still write its 503. const defaultRequestTimeout = 5 * time.Second +// Binary transfers and decoding share this budget, below the 35s socket timeout. +const defaultBinaryRequestTimeout = 30 * time.Second + const defaultIdempotencyTTL = 24 * time.Hour func New(d Deps) *App { @@ -61,7 +64,8 @@ func New(d Deps) *App { if keyTTL == 0 { keyTTL = defaultIdempotencyTTL } - chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL) + chatRepository := chatstore.NewChats(d.DB) + chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL) engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) probes := health.New( @@ -81,9 +85,15 @@ func New(d Deps) *App { auth.NewHandler(sessions).Register(public, protected) user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) - attachments := attachment.NewService(attachmentstore.NewAttachments(d.DB, d.Blobs), d.Blobs, d.MaxUploadBytes, d.MaxImagePixels) - binary := engine.Group("/api/v1", httpx.Timeout(30*time.Second), auth.RequireSession(sessions)) - attachment.NewHandler(attachments).Register(protected, binary) + attachments := attachment.NewService( + attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg, + ) + binaryTimeout := d.BinaryRequestTimeout + if binaryTimeout == 0 { + binaryTimeout = defaultBinaryRequestTimeout + } + binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions)) + attachment.NewHandler(attachments).Register(protected, binaryProtected) return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go index 2041d5e6..0b6411b7 100644 --- a/internal/app/attachment_test.go +++ b/internal/app/attachment_test.go @@ -30,32 +30,54 @@ func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) { ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized") } -func TestAttachmentMetadataUsesOrdinaryRequestDeadline(t *testing.T) { - ts := newTester(t) - actor := ts.user("deadline_user", "User") - ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, RequestTimeout: 40 * time.Millisecond}) - var held []*pgxpool.Conn - for range db.Pool.Config().MaxConns { - conn, err := db.Pool.Acquire(context.Background()) - if err != nil { - t.Fatal(err) +func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) { + for _, binary := range []bool{false, true} { + name := "metadata" + requestTimeout := 40 * time.Millisecond + var binaryTimeout time.Duration + if binary { + name = "upload" + requestTimeout = time.Second + binaryTimeout = 40 * time.Millisecond } - held = append(held, conn) - } - defer func() { - for _, conn := range held { - conn.Release() - } - }() - ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) - defer cancel() - req := httptest.NewRequest("GET", "/api/v1/attachments/11111111-1111-4111-8111-111111111111", nil).WithContext(ctx) - req.Header.Set("Authorization", "Bearer "+actor.token) - rec := httptest.NewRecorder() - started := time.Now() - ts.a.Engine.ServeHTTP(rec, req) - if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { - t.Fatalf("metadata exceeded ordinary budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + t.Run(name, func(t *testing.T) { + ts := newTester(t) + actor := ts.user("deadline_user", "User") + ts.a = app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, + SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, + RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout, + }) + var held []*pgxpool.Conn + for range db.Pool.Config().MaxConns { + conn, err := db.Pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + held = append(held, conn) + } + defer func() { + for _, conn := range held { + conn.Release() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111" + req := httptest.NewRequest("GET", path, nil).WithContext(ctx) + if binary { + path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png" + req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx) + req.Header.Set("Content-Type", "image/png") + } + req.Header.Set("Authorization", "Bearer "+actor.token) + rec := httptest.NewRecorder() + started := time.Now() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { + t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + } + }) } } diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go index 9a5cd171..69177192 100644 --- a/internal/attachment/attachment.go +++ b/internal/attachment/attachment.go @@ -1,9 +1,17 @@ package attachment import ( - "context" - "github.com/google/uuid" "time" + + "github.com/google/uuid" +) + +type FileStatus string + +const ( + FilePending FileStatus = "pending" + FileReady FileStatus = "ready" + FileFailed FileStatus = "failed" ) type Attachment struct { @@ -11,12 +19,14 @@ type Attachment struct { Filename string FileHash []byte SizeBytes int64 - MIMEType, Status string + MIMEType string + Status FileStatus CreatedAt time.Time } -type Repository interface { - IsMember(context.Context, uuid.UUID, uuid.UUID) (bool, error) - Publish(context.Context, Attachment, string) error - GetForMember(context.Context, uuid.UUID, uuid.UUID) (Attachment, error) +type UploadInput struct { + ChatID uuid.UUID + Filename string + MIMEType string + SizeBytes int64 } diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go index c211b36e..3958653b 100644 --- a/internal/attachment/blob.go +++ b/internal/attachment/blob.go @@ -16,10 +16,10 @@ type Object struct { } type Blobs interface { - Put(context.Context, string, io.Reader, int64, string) error - Open(context.Context, string) (io.ReadCloser, error) - Stat(context.Context, string) (Object, error) - Copy(context.Context, string, string) error - Delete(context.Context, string) error - List(context.Context, string, string, int) ([]Object, error) + Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error + Open(ctx context.Context, key string) (io.ReadCloser, error) + Stat(ctx context.Context, key string) (Object, error) + Copy(ctx context.Context, sourceKey, destinationKey string) error + Delete(ctx context.Context, key string) error + List(ctx context.Context, prefix, after string, limit int) ([]Object, error) } diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go index 6420519f..5adbd0e7 100644 --- a/internal/attachment/handler.go +++ b/internal/attachment/handler.go @@ -8,40 +8,52 @@ import ( "net/http" "strconv" - "github.com/gin-gonic/gin" - "github.com/google/uuid" "messenger/internal/platform/apperr" "messenger/internal/platform/httpx" "messenger/internal/platform/ident" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" ) type Handler struct{ s *Service } func NewHandler(s *Service) *Handler { return &Handler{s} } -func (h *Handler) Register(protected, binary gin.IRouter) { - binary.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) +func (h *Handler) Register(protected, binaryProtected gin.IRouter) { + binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata)) - binary.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) + binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) } type Response struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []any `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + // Lab4's shared response contract requires variants=[], even without processing. + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } func toResponse(a Attachment) Response { return Response{ - ID: a.ID.String(), ChatID: a.ChatID.String(), OwnerID: a.OwnerID.String(), Filename: a.Filename, ContentType: a.MIMEType, SizeBytes: a.SizeBytes, SHA256: hex.EncodeToString(a.FileHash), Status: a.Status, Variants: []any{}, CreatedAt: httpx.Timestamp(a.CreatedAt), UpdatedAt: httpx.Timestamp(a.CreatedAt), + ID: a.ID.String(), + ChatID: a.ChatID.String(), + OwnerID: a.OwnerID.String(), + Filename: a.Filename, + ContentType: a.MIMEType, + SizeBytes: a.SizeBytes, + SHA256: hex.EncodeToString(a.FileHash), + Status: string(a.Status), + Variants: []any{}, + CreatedAt: httpx.Timestamp(a.CreatedAt), + UpdatedAt: httpx.Timestamp(a.CreatedAt), } } @@ -65,8 +77,14 @@ func (h *Handler) upload(c *gin.Context, actor uuid.UUID) { httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) return } - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.maxBytes) - a, err := h.s.Upload(c.Request.Context(), actor, id, names[0], media, c.Request.ContentLength, c.Request.Body) + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes) + input := UploadInput{ + ChatID: id, + Filename: names[0], + MIMEType: media, + SizeBytes: c.Request.ContentLength, + } + a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body) if err != nil { httpx.Error(c, err) return @@ -109,7 +127,11 @@ func (h *Handler) content(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } - defer r.Close() + defer func() { + if err := r.Close(); err != nil { + slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err) + } + }() c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) c.Header("X-Content-Type-Options", "nosniff") c.Header("Content-Type", a.MIMEType) diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go index 729e42f6..b8a11c05 100644 --- a/internal/attachment/reconcile.go +++ b/internal/attachment/reconcile.go @@ -45,6 +45,11 @@ type Reconciler struct { grace time.Duration } +type ReconcileOptions struct { + Apply bool + Deep bool +} + func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) { if db == nil || blobs == nil { return nil, errors.New("reconcile: database and blobs are required") @@ -61,10 +66,18 @@ func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconci // Run streams findings and keeps at most one page of metadata in memory. // Dry runs only read storage. Each apply candidate is checked again under the // publisher's transaction lock; no scan snapshot authorizes a later mutation. -func (r *Reconciler) Run(ctx context.Context, apply, deep bool, emit func(Finding)) (stats Stats, err error) { +func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) { started := time.Now() defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }() - run := reconcileRun{r: r, ctx: ctx, apply: apply, deep: deep, emit: emit, stats: &stats, cutoff: started.Add(-r.grace)} + run := reconcileRun{ + r: r, + ctx: ctx, + apply: options.Apply, + deep: options.Deep, + emit: emit, + stats: &stats, + cutoff: started.Add(-r.grace), + } for _, prefix := range []string{"staging/", "blobs/"} { if err = run.objects(prefix); err != nil { return stats, err @@ -203,7 +216,8 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { return known, nil } s.stats.SQLQueries++ - rows, err := s.r.db.Query(s.ctx, `SELECT hash FROM files WHERE hash=ANY($1::bytea[])`, hashes) + rows, err := s.r.db.Query(s.ctx, ` + SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes) if err != nil { return nil, err } @@ -235,7 +249,8 @@ func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) { op.ctx = opCtx var exists bool s.stats.SQLQueries++ - if err := tx.QueryRow(opCtx, `SELECT EXISTS(SELECT 1 FROM files WHERE hash=$1)`, hash).Scan(&exists); err != nil { + if err := tx.QueryRow(opCtx, ` + SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil { return nil, err } if exists { @@ -304,7 +319,7 @@ func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Fi type reconcileFile struct { hash []byte size int64 - status string + status FileStatus created time.Time referenced bool } @@ -316,9 +331,13 @@ func (s *reconcileRun) files() error { return err } s.stats.SQLQueries++ - rows, err := s.r.db.Query(s.ctx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, - EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) - FROM files f WHERE f.hash>$1 ORDER BY f.hash LIMIT 500`, after) + rows, err := s.r.db.Query(s.ctx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash > $1 + ORDER BY f.hash + LIMIT $2`, after, reconcilePageSize) if err != nil { return err } @@ -346,30 +365,9 @@ func (s *reconcileRun) files() error { if !s.candidate(f) { continue } - var finding *Finding - if s.apply { - finding, err = s.locked(f.hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { - op := *s - op.ctx = opCtx - var current reconcileFile - s.stats.SQLQueries++ - err := tx.QueryRow(opCtx, `SELECT f.hash,f.size_bytes,f.status,f.created_at, - EXISTS(SELECT 1 FROM attachments a WHERE a.file_hash=f.hash) - FROM files f WHERE f.hash=$1 FOR UPDATE OF f`, f.hash). - Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - if !s.candidate(current) { - return nil, nil - } - return op.inspect(current, tx) - }) - } else { - finding, err = s.inspect(f, nil) + finding, err := s.inspect(f) + if err == nil && s.apply && finding != nil && finding.Action != "report" { + finding, err = s.repairFile(f.hash) } if err != nil { return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err) @@ -379,17 +377,66 @@ func (s *reconcileRun) files() error { } } +func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) { + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var current reconcileFile + s.stats.SQLQueries++ + err := tx.QueryRow(opCtx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash = $1 + FOR UPDATE OF f`, hash). + Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if !op.candidate(current) { + return nil, nil + } + finding, err := op.inspect(current) + if err != nil || finding == nil { + return finding, err + } + switch finding.Action { + case "delete_file": + // Delete bytes first. A failed SQL commit leaves an unreferenced + // missing row that the next pass can safely finish collecting. + if err := op.delete(finding.Key); err != nil { + return nil, err + } + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash) + case "mark_failed": + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed) + default: + return finding, nil + } + if err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + }) +} + func (s *reconcileRun) candidate(f reconcileFile) bool { if !s.old(f.created) { return false } if !f.referenced { - return f.status == "pending" || f.status == "ready" + return f.status == FilePending || f.status == FileReady } - return f.status == "ready" || f.status == "failed" + return f.status == FileReady || f.status == FileFailed } -func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { +func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) { key := BlobKey(f.hash) if !f.referenced { // A publication that rolled back can leave fresh bytes behind an old @@ -401,36 +448,14 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { if err == nil && !s.old(obj.Modified) { return nil, nil } - finding := &Finding{Kind: "unreferenced_" + f.status, Key: key, Action: "delete_file"} - if !s.apply { - return finding, nil - } - // S3 must succeed first. A failed SQL commit then leaves a missing, - // unreferenced row that the next run can safely finish collecting. - if err := s.delete(key); err != nil { - return nil, err - } - s.stats.SQLQueries++ - if _, err := tx.Exec(s.ctx, `DELETE FROM files WHERE hash=$1`, f.hash); err != nil { - return nil, err - } - finding.Applied = true - return finding, nil + return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil } obj, err := s.stat(key) if errors.Is(err, ErrObjectMissing) { - if f.status == "failed" { + if f.status == FileFailed { return nil, nil } - finding := &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"} - if s.apply { - s.stats.SQLQueries++ - if _, err = tx.Exec(s.ctx, `UPDATE files SET status='failed' WHERE hash=$1`, f.hash); err != nil { - return nil, err - } - finding.Applied = true - } - return finding, nil + return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil } if err != nil { return nil, err @@ -449,7 +474,7 @@ func (s *reconcileRun) inspect(f reconcileFile, tx pgx.Tx) (*Finding, error) { } // A failed file stays failed even when the key reappears. Recovery requires // an explicit publisher; a shallow HEAD cannot prove content integrity. - if f.status == "failed" { + if f.status == FileFailed { return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil } return nil, nil diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go index 39fd6dcd..d7e26884 100644 --- a/internal/attachment/reconcile_test.go +++ b/internal/attachment/reconcile_test.go @@ -209,7 +209,8 @@ func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment f.t.Fatal(err) } var found []attachment.Finding - stats, err := r.Run(context.Background(), apply, deep, func(v attachment.Finding) { found = append(found, v) }) + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) }) return stats, found, err } @@ -221,6 +222,28 @@ func TestReconciler(t *testing.T) { t.Fatal(err) } defer db.Close() + t.Run("healthy apply does not block publication", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("healthy", "ready", 2*time.Hour, true, true) + tx, err := db.Pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err := attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, f.b, time.Hour) + if err != nil { + t.Fatal(err) + } + scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) + if err != nil || stats.Findings != 0 || stats.Repaired != 0 { + t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err) + } + }) t.Run("constructor validates and defaults grace", func(t *testing.T) { f := reconcileSetup(t, db) for _, grace := range []time.Duration{-time.Second, time.Minute} { @@ -361,7 +384,7 @@ func TestReconciler(t *testing.T) { done := make(chan error, 1) go func() { r, _ := attachment.NewReconciler(f.db, f.b, 0) - _, err := r.Run(ctx, true, false, nil) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) done <- err }() waitReconcileLock(t, ctx, f.db) @@ -395,7 +418,7 @@ func TestReconciler(t *testing.T) { done := make(chan error, 1) go func() { r, _ := attachment.NewReconciler(f.db, f.b, 0) - _, err := r.Run(ctx, true, false, nil) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) done <- err }() waitReconcileLock(t, ctx, f.db) @@ -592,7 +615,7 @@ func TestReconcilerFailureBoundaries(t *testing.T) { if err != nil { t.Fatal(err) } - stats, err := r.Run(ctx, false, true, nil) + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil) if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" { t.Fatalf("read only dry run: %+v %v", stats, err) } @@ -681,11 +704,11 @@ func TestReconcilerS3(t *testing.T) { t.Fatalf("CLI mutated by default: %+v %v", cliStats, err) } var found []attachment.Finding - stats, err := r.Run(ctx, false, true, func(v attachment.Finding) { found = append(found, v) }) + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) }) if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" { t.Fatalf("S3 dry: %+v %v %+v", stats, err, found) } - stats, err = r.Run(ctx, true, true, nil) + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" { t.Fatalf("S3 apply: %+v %v", stats, err) } @@ -695,7 +718,7 @@ func TestReconcilerS3(t *testing.T) { if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil { t.Fatal("fresh staging deleted", err) } - stats, err = r.Run(ctx, true, true, nil) + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) if err != nil || stats.Repaired != 0 { t.Fatalf("S3 repeat: %+v %v", stats, err) } diff --git a/internal/attachment/repository.go b/internal/attachment/repository.go new file mode 100644 index 00000000..9c554118 --- /dev/null +++ b/internal/attachment/repository.go @@ -0,0 +1,23 @@ +package attachment + +import ( + "context" + + "github.com/google/uuid" +) + +type AttachmentRepository interface { + // WithPublication holds the file's hash lock through the callback and commit. + // Publication is valid only inside the callback. Any callback error rolls back. + WithPublication(ctx context.Context, attachment Attachment, fn func(Publication) error) error + GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error) +} + +type Publication interface { + FileReady() bool + SaveReady(ctx context.Context) error +} + +type ChatMembership interface { + IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error) +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go index ee574176..8363e1e9 100644 --- a/internal/attachment/service.go +++ b/internal/attachment/service.go @@ -15,40 +15,54 @@ import ( ) type Service struct { - repo Repository - blobs Blobs - maxBytes, maxPixels int64 - decode chan struct{} + attachments AttachmentRepository + chats ChatMembership + blobs Blobs + config Config + decode chan struct{} } -func NewService(repo Repository, blobs Blobs, maxBytes, maxPixels int64) *Service { - if maxBytes == 0 { - maxBytes = MaxUploadBytes - } - if maxPixels == 0 { - maxPixels = MaxImagePixels - } - return &Service{repo: repo, blobs: blobs, maxBytes: maxBytes, maxPixels: maxPixels, decode: make(chan struct{}, 1)} +type Config struct { + MaxBytes int64 + MaxPixels int64 } -func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, mime string, size int64, body io.Reader) (Attachment, error) { - if err := ValidateFilename(filename); err != nil { +// A nil blobs dependency allows the text API to run without configured S3. +// Binary operations report unavailability in that mode. +func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service { + if cfg.MaxBytes == 0 { + cfg.MaxBytes = MaxUploadBytes + } + if cfg.MaxPixels == 0 { + cfg.MaxPixels = MaxImagePixels + } + return &Service{ + attachments: attachments, + chats: chats, + blobs: blobs, + config: cfg, + decode: make(chan struct{}, 1), + } +} + +func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) { + if err := ValidateFilename(input.Filename); err != nil { return Attachment{}, err } - if mime != "image/png" && mime != "image/jpeg" { + if _, accepted := imageFormats[input.MIMEType]; !accepted { return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted") } - ok, err := s.repo.IsMember(ctx, chatID, actor) + ok, err := s.chats.IsMember(ctx, input.ChatID, actor) if err != nil { return Attachment{}, err } if !ok { return Attachment{}, apperr.NotFound("chat not found") } - if size > s.maxBytes { + if input.SizeBytes > s.config.MaxBytes { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } - if size <= 0 { + if input.SizeBytes <= 0 { return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required") } if s.blobs == nil { @@ -59,16 +73,17 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, defer func() { cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() + // A failed cleanup leaves a staging object for the reconciler's grace-based collection. if err := s.blobs.Delete(cleanup, stage); err != nil { - slog.Warn("staging cleanup deferred", "error", err) + slog.Warn("staging cleanup deferred", "key", stage, "error", err) } }() hash := sha256.New() - counter := &countReader{r: io.LimitReader(body, s.maxBytes+1)} - err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), size, mime) + counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)} + err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType) if err != nil { _, tooLarge := errors.AsType[*http.MaxBytesError](err) - if counter.n > s.maxBytes || tooLarge { + if counter.n > s.config.MaxBytes || tooLarge { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { @@ -78,10 +93,10 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, } var extra [1]byte n, readErr := counter.Read(extra[:]) - if counter.n > s.maxBytes { + if counter.n > s.config.MaxBytes { return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") } - if counter.n != size || n != 0 || !errors.Is(readErr, io.EOF) { + if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) { return Attachment{}, apperr.Invalid("body length does not match Content-Length") } // Full decoding is bounded by dimensions and one decoder per replica. Body @@ -91,26 +106,64 @@ func (s *Service) Upload(ctx context.Context, actor, chatID uuid.UUID, filename, case <-ctx.Done(): return Attachment{}, unavailable(ctx.Err()) } - err = s.validate(ctx, stage, mime) + err = s.validate(ctx, stage, input.MIMEType) <-s.decode if err != nil { return Attachment{}, err } - a := Attachment{ID: id, ChatID: chatID, OwnerID: actor, Filename: filename, FileHash: hash.Sum(nil), SizeBytes: size, MIMEType: mime, Status: "ready", CreatedAt: ident.TimeOf(id)} - if err := s.repo.Publish(ctx, a, stage); err != nil { + a := Attachment{ + ID: id, + ChatID: input.ChatID, + OwnerID: actor, + Filename: input.Filename, + FileHash: hash.Sum(nil), + SizeBytes: input.SizeBytes, + MIMEType: input.MIMEType, + Status: FileReady, + CreatedAt: ident.TimeOf(id), + } + if err := s.attachments.WithPublication(ctx, a, func(p Publication) error { + if err := s.publishBlob(ctx, a, stage, p.FileReady()); err != nil { + return err + } + return p.SaveReady(ctx) + }); err != nil { return Attachment{}, err } return a, nil } -func (s *Service) validate(ctx context.Context, stage, mime string) error { +func (s *Service) publishBlob(ctx context.Context, a Attachment, stagingKey string, fileReady bool) error { + blobKey := BlobKey(a.FileHash) + if fileReady { + obj, err := s.blobs.Stat(ctx, blobKey) + if err == nil && obj.Size == a.SizeBytes { + return nil + } + if err != nil && !errors.Is(err, ErrObjectMissing) { + return unavailable(err) + } + } + // The hash is known after streaming. Copy the validated staging object to + // its permanent CAS key while the repository's publication lock is held. + if err := s.blobs.Copy(ctx, stagingKey, blobKey); err != nil { + return unavailable(err) + } + return nil +} + +func (s *Service) validate(ctx context.Context, stage, mime string) (err error) { r, err := s.blobs.Open(ctx, stage) if err != nil { return unavailable(err) } - defer r.Close() + defer func() { + if closeErr := r.Close(); closeErr != nil && err == nil { + err = unavailable(closeErr) + } + }() tracked := &storageReader{reader: r} - err = ValidateImage(tracked, mime, s.maxPixels) + err = ValidateImage(tracked, mime, s.config.MaxPixels) if tracked.err != nil { return unavailable(tracked.err) } @@ -136,11 +189,11 @@ func (r *storageReader) Read(p []byte) (int, error) { } func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) { - a, err := s.repo.GetForMember(ctx, id, actor) + a, err := s.attachments.GetForMember(ctx, id, actor) if err != nil { return Attachment{}, err } - if a.Status != "ready" { + if a.Status != FileReady { return Attachment{}, apperr.Conflict("attachment is not ready") } return a, nil @@ -166,7 +219,12 @@ type countReader struct { n int64 } -func (c *countReader) Read(b []byte) (int, error) { n, e := c.r.Read(b); c.n += int64(n); return n, e } +func (c *countReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.n += int64(n) + return n, err +} + func unavailable(err error) error { return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) } diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go index c207a956..f01a3540 100644 --- a/internal/attachment/service_test.go +++ b/internal/attachment/service_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" "messenger/internal/attachment" "messenger/internal/attachment/store" + chatstore "messenger/internal/chat/store" "messenger/internal/platform/apperr" "messenger/internal/platform/ident" "messenger/internal/testenv" @@ -73,10 +74,11 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { t.Fatal(err) } b := &blockingPublication{testBlobs: &testBlobs{data: map[string][]byte{}}, entered: make(chan struct{}), resume: make(chan struct{})} - s := attachment.NewService(store.NewAttachments(db.Pool, b), b, 0, 0) + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{}) uploaded := make(chan error, 1) go func() { - _, e := s.Upload(ctx, actor, room, "race.png", "image/png", int64(len(data)), bytes.NewReader(data)) + input := attachment.UploadInput{ChatID: room, Filename: "race.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + _, e := s.Upload(ctx, actor, input, bytes.NewReader(data)) uploaded <- e }() select { @@ -90,7 +92,7 @@ func TestCollectorRechecksReferencesAfterConcurrentPublication(t *testing.T) { } collected := make(chan error, 1) go func() { - stats, e := r.Run(ctx, true, false, nil) + stats, e := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) if e == nil && stats.Repaired != 0 { e = errors.New("collector removed a newly referenced file") } @@ -196,7 +198,7 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { } } b := &testBlobs{data: map[string][]byte{}} - s := attachment.NewService(store.NewAttachments(db.Pool, b), b, attachment.MaxUploadBytes, attachment.MaxImagePixels) + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{}) data, err := os.ReadFile("../../tests/fixtures/sample.png") if err != nil { t.Fatal(err) @@ -210,7 +212,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { c = c2 } var e error - results[i], e = s.Upload(ctx, actor, c, "photo.png", "image/png", int64(len(data)), bytes.NewReader(data)) + input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data)) if e != nil { t.Error(e) } @@ -235,7 +238,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { b.mu.Lock() clear(b.data) b.mu.Unlock() - if _, err := s.Upload(ctx, actor, c1, "again.png", "image/png", int64(len(data)), bytes.NewReader(data)); err != nil { + input := attachment.UploadInput{ChatID: c1, Filename: "again.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(data)); err != nil { t.Fatal(err) } if b.copies != 2 { @@ -244,7 +248,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { b.failCopy = true other := bytes.Clone(data) other = append(other, 0) - if _, err := s.Upload(ctx, actor, c1, "failed.png", "image/png", int64(len(other)), bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { + input.Filename, input.SizeBytes = "failed.png", int64(len(other)) + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); apperr.KindOf(err) != apperr.KindUnavailable { t.Fatalf("copy outage: %v", err) } if len(b.data) != 1 { @@ -258,7 +263,8 @@ func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.Upload(ctx, actor, c1, "rollback.png", "image/png", int64(len(other)), bytes.NewReader(other)); err == nil { + input.Filename = "rollback.png" + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil { t.Fatal("publication survived rejected metadata") } if len(b.data) != 2 { diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go index 607932a9..32105070 100644 --- a/internal/attachment/store/postgres.go +++ b/internal/attachment/store/postgres.go @@ -4,32 +4,29 @@ import ( "context" "errors" "fmt" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" ) type Attachments struct { - pool *pgxpool.Pool - blobs attachment.Blobs + pool *pgxpool.Pool } -func NewAttachments(pool *pgxpool.Pool, blobs attachment.Blobs) *Attachments { - return &Attachments{pool, blobs} -} +var _ attachment.AttachmentRepository = (*Attachments)(nil) -func (r *Attachments) IsMember(ctx context.Context, chatID, actor uuid.UUID) (bool, error) { - var ok bool - err := r.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, actor).Scan(&ok) - return ok, postgres.Classify(err) +func NewAttachments(pool *pgxpool.Pool) *Attachments { + return &Attachments{pool: pool} } // Publication and garbage collection use the same hash lock. The transaction // spans only publication, never client transfer or image decoding. -func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stage string) error { +func (r *Attachments) WithPublication(ctx context.Context, a attachment.Attachment, fn func(attachment.Publication) error) error { return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { if _, err := tx.Exec(ctx, `SET LOCAL idle_in_transaction_session_timeout = '35s'`); err != nil { return err @@ -37,41 +34,56 @@ func (r *Attachments) Publish(ctx context.Context, a attachment.Attachment, stag if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil { return err } - var status string - err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1 FOR UPDATE`, a.FileHash).Scan(&status) + var status attachment.FileStatus + err := tx.QueryRow(ctx, `SELECT status FROM files WHERE hash = $1 FOR UPDATE`, a.FileHash).Scan(&status) if err != nil && !errors.Is(err, pgx.ErrNoRows) { return err } - copyNeeded := status != "ready" - if !copyNeeded { - obj, e := r.blobs.Stat(ctx, attachment.BlobKey(a.FileHash)) - if errors.Is(e, attachment.ErrObjectMissing) { - copyNeeded = true - } else if e != nil { - return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", e) - } else if obj.Size != a.SizeBytes { - copyNeeded = true - } - } - if copyNeeded { - if err := r.blobs.Copy(ctx, stage, attachment.BlobKey(a.FileHash)); err != nil { - return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) - } - } - _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,$2,$3,'ready') ON CONFLICT(hash) DO UPDATE SET size_bytes=excluded.size_bytes,mime_type=excluded.mime_type,status='ready'`, a.FileHash, a.SizeBytes, a.MIMEType) - if err != nil { - return fmt.Errorf("publish file: %w", err) - } - _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) - return err + return fn(publication{tx: tx, attachment: a, ready: status == attachment.FileReady}) })) } +type publication struct { + tx pgx.Tx + attachment attachment.Attachment + ready bool +} + +func (p publication) FileReady() bool { return p.ready } + +func (p publication) SaveReady(ctx context.Context) error { + a := p.attachment + _, err := p.tx.Exec(ctx, ` + INSERT INTO files (hash, size_bytes, mime_type, status) + VALUES ($1, $2, $3, $4) + ON CONFLICT (hash) DO UPDATE + SET size_bytes = excluded.size_bytes, + mime_type = excluded.mime_type, + status = excluded.status`, + a.FileHash, a.SizeBytes, a.MIMEType, attachment.FileReady) + if err != nil { + return fmt.Errorf("publish file: %w", err) + } + _, err = p.tx.Exec(ctx, ` + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + VALUES ($1, $2, $3, $4, $5, $6)`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt) + return err +} + func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) { var a attachment.Attachment - err := r.pool.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.file_hash,f.size_bytes,f.mime_type,f.status,a.created_at - FROM attachments a JOIN files f ON f.hash=a.file_hash - WHERE a.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=$2)`, id, actor).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) + err := r.pool.QueryRow(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash, + f.size_bytes, f.mime_type, f.status, a.created_at + FROM attachments a + JOIN files f ON f.hash = a.file_hash + WHERE a.id = $1 AND EXISTS ( + SELECT 1 FROM chat_members m + WHERE m.chat_id = a.chat_id AND m.user_id = $2 + )`, id, actor). + Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, + &a.SizeBytes, &a.MIMEType, &a.Status, &a.CreatedAt) if errors.Is(err, pgx.ErrNoRows) { return attachment.Attachment{}, apperr.NotFound("attachment not found") } diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go index a810545f..02ec9b80 100644 --- a/internal/attachment/validation.go +++ b/internal/attachment/validation.go @@ -16,6 +16,11 @@ import ( const MaxUploadBytes int64 = 10 * 1024 * 1024 const MaxImagePixels int64 = 40_000_000 +var imageFormats = map[string]string{ + "image/png": "png", + "image/jpeg": "jpeg", +} + func ValidateFilename(name string) error { if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { return apperr.Invalid("filename must contain 1-255 characters") @@ -37,7 +42,7 @@ func ValidateImage(r io.Reader, mime string, maxPixels int64) error { if err != nil { return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) } - if (format != "png" && format != "jpeg") || "image/"+format != mime { + if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat { return apperr.UnsupportedMedia("image does not match Content-Type") } if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) { diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go index 7ef3c509..0c7b4092 100644 --- a/internal/attachment/validation_test.go +++ b/internal/attachment/validation_test.go @@ -8,11 +8,44 @@ import ( "image/color" "image/png" "io" - "messenger/internal/platform/apperr" + "os" "strings" "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" ) +type chatMember struct { + chatID, userID uuid.UUID +} + +func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) { + return chatID == m.chatID && userID == m.userID, nil +} + +func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) { + member := chatMember{chatID: uuid.New(), userID: uuid.New()} + s := NewService(nil, member, nil, Config{}) + input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4} + for _, tc := range []struct { + name string + actor uuid.UUID + kind apperr.Kind + }{ + {name: "member", actor: member.userID, kind: apperr.KindUnavailable}, + {name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data")) + if apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want %v", err, tc.kind) + } + }) + } +} + func TestValidateFilename(t *testing.T) { for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} { if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid { @@ -41,16 +74,35 @@ type openedBlob struct { } func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) { + if reader, ok := b.reader.(io.ReadCloser); ok { + return reader, nil + } return io.NopCloser(b.reader), nil } +type closeErrorBody struct{ io.Reader } + +func (closeErrorBody) Close() error { return errors.New("injected close failure") } + +func TestStagingCloseFailurePreventsPublication(t *testing.T) { + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}} + s := NewService(nil, nil, blobs, Config{}) + if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("close failure classified as %v", err) + } +} + func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) { var pngData bytes.Buffer if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil { t.Fatal(err) } for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} { - s := NewService(nil, openedBlob{reader: &failingRead{prefix: prefix}}, 0, 0) + s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{}) err := s.validate(context.Background(), "staging/test", "image/png") if apperr.KindOf(err) != apperr.KindUnavailable { t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err) diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index cd6df783..4af60ff1 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -18,6 +18,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/attachment" "messenger/internal/chat" "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" @@ -202,7 +203,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [ if err != nil { return fmt.Errorf("check attachments: %w", err) } - statuses, err := pgx.CollectRows(rows, pgx.RowTo[string]) + statuses, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus]) if err != nil { return fmt.Errorf("read attachment status: %w", err) } @@ -210,7 +211,7 @@ func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids [ return apperr.NotFound("attachment not found") } for _, status := range statuses { - if status != "ready" { + if status != attachment.FileReady { return apperr.Conflict("attachment is not ready") } } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 10b4b523..2a1e485b 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -134,7 +134,7 @@ func Load(lookup Lookup) (Config, error) { } cfg.DebugAddr = v } - if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { return Config{}, err } if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { @@ -154,12 +154,16 @@ func LoadMigrate(lookup Lookup) (Migrate, error) { if cfg.Logging, err = loadLogging(lookup); err != nil { return Migrate{}, err } - if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { return Migrate{}, err } return cfg, nil } +func LoadDatabaseURL(lookup Lookup) (string, error) { + return secret(lookup, "DATABASE_URL_FILE") +} + func loadLogging(lookup Lookup) (Logging, error) { l := Logging{Env: "lab", Level: slog.LevelInfo} if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go index 7a6385e9..d68429ff 100644 --- a/internal/platform/s3store/bootstrap_test.go +++ b/internal/platform/s3store/bootstrap_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -85,7 +86,7 @@ func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) { } })) defer server.Close() - s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -126,7 +127,7 @@ func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) { io.WriteString(w, `ServiceUnavailablevolume not registered`) })) defer server.Close() - s, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go index 9798a581..4015511b 100644 --- a/internal/platform/s3store/integration_test.go +++ b/internal/platform/s3store/integration_test.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "messenger/internal/attachment" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" "messenger/internal/testenv" ) @@ -149,7 +150,7 @@ func TestSeaweedStore(t *testing.T) { t.Fatal(err) } for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} { - outside, err := s3store.New(s.Endpoint, s.Region, "private-other", pair[0], pair[1]) + outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]}) if err != nil { t.Fatal(err) } @@ -219,7 +220,7 @@ func TestSeaweedStore(t *testing.T) { t.Fatal(err) } } - missingBucket, err := s3store.New(s.Endpoint, s.Region, "absent-bucket", s.AdminAccessKey, s.AdminSecretKey) + missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) if err != nil { t.Fatal(err) } diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go index 2c4b9b9e..86d1bb0f 100644 --- a/internal/platform/s3store/store.go +++ b/internal/platform/s3store/store.go @@ -16,10 +16,13 @@ import ( "github.com/minio/minio-go/v7/pkg/s3utils" "messenger/internal/attachment" + "messenger/internal/platform/config" ) const MaxPageSize = 500 +const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" + type Store struct { client *minio.Core region string @@ -30,15 +33,15 @@ var _ attachment.Blobs = (*Store)(nil) // New validates configuration without contacting S3, so text-only service startup // and readiness do not depend on object storage being reachable. -func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) { - u, err := url.Parse(endpoint) +func New(cfg config.S3) (*Store, error) { + u, err := url.Parse(cfg.Endpoint) if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment") } - if strings.TrimSpace(region) == "" || accessKey == "" || secretKey == "" { + if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { return nil, errors.New("s3: region and credentials are required") } - if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil { + if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil { return nil, fmt.Errorf("s3: invalid bucket: %w", err) } transport := &http.Transport{ @@ -56,9 +59,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) DisableCompression: true, } client, err := minio.NewCore(u.Host, &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), Secure: u.Scheme == "https", - Region: region, + Region: cfg.Region, BucketLookup: minio.BucketLookupPath, Transport: transport, // A consumed upload stream cannot be replayed. Let the caller decide @@ -68,7 +71,7 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Store, error) if err != nil { return nil, fmt.Errorf("s3: client: %w", err) } - return &Store{client: client, region: region, bucket: bucket}, nil + return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil } // Bootstrap is used only by the admin init command, never by API startup. @@ -78,7 +81,7 @@ func (s *Store) Bootstrap(ctx context.Context) error { } err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region}) code := minio.ToErrorResponse(err).Code - if err != nil && code != "BucketAlreadyOwnedByYou" { + if err != nil && code != codeBucketAlreadyOwnedByYou { return wrap("create bucket", err) } return s.WaitReady(ctx) diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go index cac6af56..41fb2383 100644 --- a/internal/platform/s3store/store_test.go +++ b/internal/platform/s3store/store_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -18,7 +19,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) })) defer server.Close() - if _, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret"); err != nil { + if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil { t.Fatal(err) } if calls.Load() != 0 { @@ -36,7 +37,7 @@ func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { {"secret", server.URL, "us-east-1", "attachments", "access", ""}, } { t.Run(tc.name, func(t *testing.T) { - if _, err := s3store.New(tc.endpoint, tc.region, tc.bucket, tc.access, tc.secret); err == nil { + if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil { t.Fatal("accepted invalid configuration") } }) @@ -69,7 +70,7 @@ func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) { w.Header().Set("ETag", `"probe"`) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -103,7 +104,7 @@ func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } @@ -154,7 +155,7 @@ func TestListStopsAfterOneBoundedPage(t *testing.T) { io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`) })) defer server.Close() - store, err := s3store.New(server.URL, "us-east-1", "attachments", "access", "secret") + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) if err != nil { t.Fatal(err) } diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go index 062c9f99..36e551bf 100644 --- a/internal/testenv/s3.go +++ b/internal/testenv/s3.go @@ -22,6 +22,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "messenger/internal/platform/config" "messenger/internal/platform/s3store" ) @@ -131,15 +132,15 @@ func (s *S3) launch(ctx context.Context) error { if err != nil { return err } - s.Store, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AccessKey, s.SecretKey) + s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey}) if err != nil { return err } - s.APIStore, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.APIAccessKey, s.APISecretKey) + s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey}) if err != nil { return err } - s.Admin, err = s3store.New(s.Endpoint, s.Region, s.Bucket, s.AdminAccessKey, s.AdminSecretKey) + s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) return err } diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go index b1b3ab47..2c945867 100644 --- a/tools/reconcilebench/main.go +++ b/tools/reconcilebench/main.go @@ -119,7 +119,8 @@ func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, } }() m := measurement{Phase: phase, Mode: mode, TargetFiles: n} - stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) close(done) m.GoHeapPeakBytes = <-peakCh m.Stats = stats -- 2.54.0