From 5b9f3777836b39c521349392760bf20b77378046 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 10:00:11 +0300 Subject: [PATCH 01/21] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=B5?= =?UTF-8?q?=D1=81=D1=82=D0=B8=20=D1=80=D0=B5=D1=88=D0=B5=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=202=20=D0=BA=D0=B0=D0=BA=20=D0=BE=D1=81=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D1=83=20=D0=B4=D0=BB=D1=8F=20=D0=BB=D0=B0=D0=B1?= =?UTF-8?q?=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD=D0=BE=D0=B9=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .gitattributes | 6 + .github/workflows/app.yml | 52 ++ Dockerfile | 20 + Makefile | 40 ++ cmd/server/main.go | 78 +++ compose.yaml | 84 ++++ go.mod | 107 ++++ go.sum | 304 ++++++++++++ infra/valkey/valkey.conf | 19 + internal/app/app.go | 70 +++ internal/app/app_test.go | 510 ++++++++++++++++++++ internal/app/auth_test.go | 162 +++++++ internal/app/failure_test.go | 100 ++++ internal/auth/handler.go | 89 ++++ internal/auth/main_test.go | 33 ++ internal/auth/middleware.go | 48 ++ internal/auth/repository.go | 18 + internal/auth/service.go | 64 +++ internal/auth/service_test.go | 145 ++++++ internal/auth/session.go | 39 ++ internal/auth/store/valkey.go | 64 +++ internal/chat/chat.go | 82 ++++ internal/chat/chat_test.go | 88 ++++ internal/chat/handler.go | 192 ++++++++ internal/chat/message.go | 44 ++ internal/chat/message_test.go | 47 ++ internal/chat/repository.go | 31 ++ internal/chat/service.go | 101 ++++ internal/chat/service_test.go | 234 +++++++++ internal/chat/store/postgres.go | 153 ++++++ internal/chat/store/postgres_test.go | 313 ++++++++++++ internal/health/handler.go | 77 +++ internal/platform/apperr/apperr.go | 54 +++ internal/platform/apperr/apperr_test.go | 49 ++ internal/platform/config/config.go | 118 +++++ internal/platform/config/config_test.go | 135 ++++++ internal/platform/httpx/decode.go | 70 +++ internal/platform/httpx/engine.go | 49 ++ internal/platform/httpx/httpx_test.go | 376 +++++++++++++++ internal/platform/httpx/identity.go | 42 ++ internal/platform/httpx/middleware.go | 118 +++++ internal/platform/httpx/respond.go | 87 ++++ internal/platform/ident/ident.go | 46 ++ internal/platform/ident/ident_test.go | 72 +++ internal/platform/pagination/cursor.go | 63 +++ internal/platform/pagination/cursor_test.go | 71 +++ internal/platform/pagination/page.go | 41 ++ internal/platform/pagination/page_test.go | 60 +++ internal/platform/password/password.go | 74 +++ internal/platform/password/password_test.go | 78 +++ internal/platform/postgres/postgres.go | 145 ++++++ internal/platform/postgres/postgres_test.go | 195 ++++++++ internal/platform/valkey/valkey.go | 49 ++ internal/platform/valkey/valkey_test.go | 57 +++ internal/server/server.go | 60 +++ internal/server/server_test.go | 102 ++++ internal/testenv/testenv.go | 181 +++++++ internal/testenv/testenv_test.go | 29 ++ internal/user/handler.go | 70 +++ internal/user/main_test.go | 24 + internal/user/repository.go | 18 + internal/user/service.go | 88 ++++ internal/user/service_test.go | 189 ++++++++ internal/user/store/postgres.go | 74 +++ internal/user/user.go | 75 +++ internal/user/user_test.go | 78 +++ migrations/00001_init.sql | 42 ++ migrations/embed.go | 8 + scripts/explain-history.sql | 75 +++ scripts/init-secrets.sh | 42 ++ scripts/restart.cmd | 5 + scripts/restart.sh | 9 + tools/concurrency/client.go | 83 ++++ tools/concurrency/main.go | 227 +++++++++ tools/concurrency/main_test.go | 65 +++ 76 files changed, 7008 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 cmd/server/main.go create mode 100644 compose.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/valkey/valkey.conf create mode 100644 internal/app/app.go create mode 100644 internal/app/app_test.go create mode 100644 internal/app/auth_test.go create mode 100644 internal/app/failure_test.go create mode 100644 internal/auth/handler.go create mode 100644 internal/auth/main_test.go create mode 100644 internal/auth/middleware.go create mode 100644 internal/auth/repository.go create mode 100644 internal/auth/service.go create mode 100644 internal/auth/service_test.go create mode 100644 internal/auth/session.go create mode 100644 internal/auth/store/valkey.go create mode 100644 internal/chat/chat.go create mode 100644 internal/chat/chat_test.go create mode 100644 internal/chat/handler.go create mode 100644 internal/chat/message.go create mode 100644 internal/chat/message_test.go create mode 100644 internal/chat/repository.go create mode 100644 internal/chat/service.go create mode 100644 internal/chat/service_test.go create mode 100644 internal/chat/store/postgres.go create mode 100644 internal/chat/store/postgres_test.go create mode 100644 internal/health/handler.go create mode 100644 internal/platform/apperr/apperr.go create mode 100644 internal/platform/apperr/apperr_test.go create mode 100644 internal/platform/config/config.go create mode 100644 internal/platform/config/config_test.go create mode 100644 internal/platform/httpx/decode.go create mode 100644 internal/platform/httpx/engine.go create mode 100644 internal/platform/httpx/httpx_test.go create mode 100644 internal/platform/httpx/identity.go create mode 100644 internal/platform/httpx/middleware.go create mode 100644 internal/platform/httpx/respond.go create mode 100644 internal/platform/ident/ident.go create mode 100644 internal/platform/ident/ident_test.go create mode 100644 internal/platform/pagination/cursor.go create mode 100644 internal/platform/pagination/cursor_test.go create mode 100644 internal/platform/pagination/page.go create mode 100644 internal/platform/pagination/page_test.go create mode 100644 internal/platform/password/password.go create mode 100644 internal/platform/password/password_test.go create mode 100644 internal/platform/postgres/postgres.go create mode 100644 internal/platform/postgres/postgres_test.go create mode 100644 internal/platform/valkey/valkey.go create mode 100644 internal/platform/valkey/valkey_test.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go create mode 100644 internal/testenv/testenv.go create mode 100644 internal/testenv/testenv_test.go create mode 100644 internal/user/handler.go create mode 100644 internal/user/main_test.go create mode 100644 internal/user/repository.go create mode 100644 internal/user/service.go create mode 100644 internal/user/service_test.go create mode 100644 internal/user/store/postgres.go create mode 100644 internal/user/user.go create mode 100644 internal/user/user_test.go create mode 100644 migrations/00001_init.sql create mode 100644 migrations/embed.go create mode 100644 scripts/explain-history.sql create mode 100755 scripts/init-secrets.sh create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 tools/concurrency/client.go create mode 100644 tools/concurrency/main.go create mode 100644 tools/concurrency/main_test.go diff --git a/.dockerignore b/.dockerignore index bc16e556..c872c535 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +compose*.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..1af1a8fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Files read inside Linux containers or by sh must keep LF on Windows +# checkouts with core.autocrlf=true. +*.sh text eol=lf +*.conf text eol=lf +*.sql text eol=lf +*.cmd text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..a27b1206 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,52 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +defaults: + run: + # Explicit bash turns on `-eo pipefail`, so a failing check behind `| tee` fails the job. + shell: bash +env: + BASE_URL: http://localhost:8080 +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - run: go vet ./... + - run: go test -race -count=1 ./... + e2e: + runs-on: ubuntu-latest + needs: unit + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: make secrets + - run: docker compose up -d --build --wait + - run: mkdir -p evidence-ci + - name: Public smoke + run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt + - name: Persistence across a restart of the API and both stores + run: make test-persistence PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/restart.sh | tee evidence-ci/persistence.txt + - name: Concurrency check + run: go run ./tools/concurrency --base-url "$BASE_URL" --output evidence-ci/concurrency.json + - name: Container logs + if: always() + run: docker compose logs > evidence-ci/compose.log 2>&1 || true + - name: Cleanup + # The CI stand is disposable, so its volumes go too. + if: always() + run: docker compose down -v || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: evidence + path: evidence-ci diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b8bc70b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Build stage: compile a static binary with the exact toolchain pinned in go.mod. +FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download && go mod verify +COPY . . +RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +# Runtime stage: minimal image, unprivileged user, the server is PID 1. +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 +RUN adduser -D -H -u 10001 app +COPY --from=build /out/server /usr/local/bin/server +USER app +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 \ + CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 +ENTRYPOINT ["/usr/local/bin/server"] diff --git a/Makefile b/Makefile index 5d86d171..63093d80 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,43 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --------------------------------------------------------------------------- +# Application targets (Go implementation). The targets above are the skeleton. +# --------------------------------------------------------------------------- +EVIDENCE_DIR ?= evidence + +.PHONY: build vet unit secrets up down destroy logs concurrency explain + +build: + go build -o bin/server ./cmd/server + +vet: + go vet ./... + +# The tests start disposable PostgreSQL and Valkey containers: Docker must run. +unit: + go test -race -count=1 ./... + +secrets: + sh scripts/init-secrets.sh + +up: + docker compose up -d --build --wait + +# Stops and removes the containers; volumes and therefore data are kept. +down: + docker compose down + +# Deletes the volumes too: every user, chat, message and session of the stand. +destroy: + docker compose down -v + +logs: + docker compose logs + +concurrency: + go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" + +explain: + docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 00000000..305e4ff2 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,78 @@ +// Command server starts the messenger API over PostgreSQL and Valkey. +package main + +import ( + "context" + "fmt" + "log/slog" + "net" + "os" + "os/signal" + "syscall" + + "messenger/internal/app" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/valkey" + "messenger/internal/server" +) + +func main() { + cfg, err := config.Load(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + log := newLogger(cfg) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg); err != nil { + log.Error("server stopped with error", "error", err.Error()) + os.Exit(1) + } +} + +// run connects the dependencies before it listens, so the port accepts +// connections only once the schema is in place. +func run(ctx context.Context, log *slog.Logger, cfg config.Config) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL) + if err != nil { + return err + } + defer pool.Close() + applied, err := postgres.Migrate(ctx, pool) + if err != nil { + return err + } + log.Info("schema ready", "applied_migrations", applied) + sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL) + if err != nil { + return err + } + defer sessions.Close() + + ln, err := net.Listen("tcp", cfg.Addr()) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.Addr(), err) + } + log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(), + "shutdown_timeout", cfg.ShutdownTimeout.String(), "session_ttl", cfg.SessionTTL.String()) + a := app.New(app.Deps{ + Log: log, + DB: pool, + Sessions: sessions, + SessionTTL: cfg.SessionTTL, + MaxBodyBytes: cfg.MaxBodyBytes, + }) + return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.ShutdownTimeout) +} + +func newLogger(cfg config.Config) *slog.Logger { + opts := &slog.HandlerOptions{Level: cfg.LogLevel} + if cfg.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0a11d1f6 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,84 @@ +# Local stand: the API over PostgreSQL (users, chats, messages) and Valkey +# (sessions). Only the API is published, on loopback. Credentials are files in +# ./secrets created by `make secrets`; named volumes keep the data across +# restarts, and only `make destroy` removes them. +services: + api: + build: . + image: mtusi-messenger:lab3 + # Fixed name for docker logs/inspect in the report. + container_name: mtusi-messenger-lab3 + env_file: + - path: .env + required: false + environment: + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_STORE_URL_FILE: /run/secrets/session_store_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: + - database_url + - session_store_url + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + # Loopback only; override with PUBLISH=127.0.0.1:18080 when 8080 is busy. + ports: + - "${PUBLISH:-127.0.0.1:8080}:8080" + stop_grace_period: 10s + + postgres: + image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + # Since PostgreSQL 18 the image keeps PGDATA in a versioned directory + # below /var/lib/postgresql, so the volume is mounted on the parent. + volumes: + - pgdata:/var/lib/postgresql + healthcheck: + # Over TCP: during first initialisation the image runs a temporary + # server on the Unix socket only, which must not count as ready. + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + # A clean shutdown writes a checkpoint instead of leaving crash recovery + # to the next start. + stop_grace_period: 30s + + valkey: + image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + secrets: + - valkey_auth + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + - valkeydata:/data + healthcheck: + # valkey-cli exits 0 even on NOAUTH, so the reply itself is checked. + test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + stop_grace_period: 10s + +volumes: + pgdata: + valkeydata: + +secrets: + database_url: + file: ./secrets/database_url + session_store_url: + file: ./secrets/session_store_url + postgres_password: + file: ./secrets/postgres_password + valkey_auth: + file: ./secrets/valkey_auth.conf diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..9c6fb13e --- /dev/null +++ b/go.mod @@ -0,0 +1,107 @@ +module messenger + +go 1.27.1 + +require ( + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/pressly/goose/v3 v3.27.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/testcontainers/testcontainers-go v0.39.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 + github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 + golang.org/x/crypto v0.57.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/creack/pty v1.1.24 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.3.3+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/stretchr/testify v1.12.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..fb12767b --- /dev/null +++ b/go.sum @@ -0,0 +1,304 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.27.0 h1:/D30gVTuQhu0WsNZYbJi4DMOsx1lNq+6SkLe+Wp59BM= +github.com/pressly/goose/v3 v3.27.0/go.mod h1:3ZBeCXqzkgIRvrEMDkYh1guvtoJTU5oMMuDdkutoM78= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= +github.com/testcontainers/testcontainers-go v0.39.0/go.mod h1:qmHpkG7H5uPf/EvOORKvS6EuDkBUPE3zpVGaH9NL7f8= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/valkey-io/valkey-go v1.0.59 h1:W67Z0UY+Qqk3k8NKkFCFlM3X4yQUniixl7dSJAch2Qo= +github.com/valkey-io/valkey-go v1.0.59/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ= +modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..d7e96bff --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,19 @@ +# Valkey is the only store of user sessions, so it is their source of truth. +# Every write is fsynced before the reply: an acknowledged login survives a +# crash or a restart. Only logins and logouts write; session lookups are reads +# and pay nothing for this. +appendonly yes +appendfsync always +dir /data + +# The append-only file already persists everything; RDB snapshots are off. +save "" + +# Sessions are never evicted to free memory: once maxmemory is reached the +# store refuses new sessions (login answers 503) instead of silently logging +# users out. A session key takes about a hundred bytes. +maxmemory 256mb +maxmemory-policy noeviction + +# requirepass comes from the compose secret, never from the repository. +include /run/secrets/valkey_auth diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 00000000..6653c8c5 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,70 @@ +// Package app is the composition root: it builds every module with its +// stores and mounts them on one HTTP engine. Both the server binary and the +// integration tests start from here. +package app + +import ( + "context" + "log/slog" + "time" + + "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/chat" + chatstore "messenger/internal/chat/store" + "messenger/internal/health" + "messenger/internal/platform/httpx" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +type App struct { + Engine *gin.Engine + Health *health.Handler +} + +// Deps are the shared resources the modules are built on; their lifetime is +// owned by the caller. +type Deps struct { + Log *slog.Logger + DB *pgxpool.Pool + Sessions *redis.Client + SessionTTL time.Duration + MaxBodyBytes int64 + // RequestTimeout bounds every API request; zero means the default. + RequestTimeout time.Duration +} + +// defaultRequestTimeout stays below the server's 10s write timeout, so a +// request that gives up on a dependency can still write its 503. +const defaultRequestTimeout = 5 * time.Second + +func New(d Deps) *App { + users := user.NewService(userstore.NewUsers(d.DB)) + chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users) + + engine := httpx.NewEngine(d.Log, d.MaxBodyBytes) + probes := health.New( + health.Check{Name: "postgres", Probe: d.DB.Ping}, + health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, + ) + probes.Register(engine) + + sessions := auth.NewService(users, authstore.NewSessions(d.Sessions), d.SessionTTL) + + timeout := d.RequestTimeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + public := engine.Group("/api/v1", httpx.Timeout(timeout)) + protected := public.Group("", auth.RequireSession(sessions)) + auth.NewHandler(sessions).Register(public, protected) + user.NewHandler(users).Register(public, protected) + chat.NewHandler(chats).Register(protected) + + return &App{Engine: engine, Health: probes} +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 00000000..b9ba72b6 --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,510 @@ +package app_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sort" + "strings" + "sync" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/testenv" + "messenger/internal/user" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +const testPassword = "correct-horse-battery" + +// errorBody mirrors the contract error envelope. +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type response struct { + status int + requestID string + challenge string + raw []byte +} + +// tester drives the assembled application in-process. +type tester struct { + t *testing.T + a *app.App +} + +// account is a registered user with a live session. +type account struct { + user.Response + token string +} + +// sessionBody mirrors the contract Session schema. +type sessionBody struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +func newTester(t *testing.T) *tester { + t.Helper() + return newTesterWithTTL(t, time.Hour) +} + +// newTesterWithTTL assembles the application over empty stores. +func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester { + t.Helper() + db.Reset(t) + kv.Reset(t) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + return &tester{t: t, a: app.New(app.Deps{ + Log: log, + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: ttl, + MaxBodyBytes: 64 * 1024, + })} +} + +// do sends one request, authenticated with the bearer token when it is set. +// A string body is sent verbatim, anything else is JSON-encoded; both get a +// JSON content type. Every response is checked for X-Request-Id, error +// bodies for the contract envelope, and every 401 for the challenge of the +// scheme the route expects. +func (ts *tester) do(method, path, token string, body any, headers map[string]string) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, encode(ts.t, body)) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + + res := response{ + status: rec.Code, + requestID: rec.Header().Get("X-Request-Id"), + challenge: rec.Header().Get("WWW-Authenticate"), + raw: rec.Body.Bytes(), + } + if res.requestID == "" { + ts.t.Fatalf("%s %s: missing X-Request-Id", method, path) + } + if res.status >= 400 { + var eb errorBody + if err := json.Unmarshal(res.raw, &eb); err != nil { + ts.t.Fatalf("%s %s: error body is not the contract envelope: %v: %s", method, path, err, res.raw) + } + if eb.Error.RequestID != res.requestID || eb.Error.Code == "" || eb.Error.Message == "" { + ts.t.Fatalf("%s %s: bad error envelope: %s", method, path, res.raw) + } + } + if res.status == http.StatusUnauthorized { + scheme := "Bearer " + if path == "/api/v1/auth/sessions" { + scheme = "Basic " + } + if !strings.HasPrefix(res.challenge, scheme) { + ts.t.Fatalf("%s %s: 401 challenge = %q, want scheme %q", method, path, res.challenge, scheme) + } + } + return res +} + +func encode(t *testing.T, body any) io.Reader { + t.Helper() + switch b := body.(type) { + case nil: + return nil + case string: + return strings.NewReader(b) + default: + raw, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + return bytes.NewReader(raw) + } +} + +// expect asserts the status and, for errors, the contract code. +func (ts *tester) expect(res response, status int, code string) { + ts.t.Helper() + if res.status != status { + ts.t.Fatalf("status = %d, want %d; body %s", res.status, status, res.raw) + } + if code != "" { + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + if eb.Error.Code != code { + ts.t.Fatalf("code = %q, want %q; body %s", eb.Error.Code, code, res.raw) + } + } +} + +// decode parses a success body into a typed DTO and rejects unknown fields, +// which enforces the additionalProperties=false rule of the contract. +func decode[T any](t *testing.T, res response) T { + t.Helper() + var out T + dec := json.NewDecoder(bytes.NewReader(res.raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + t.Fatalf("body does not match %T: %v: %s", out, err, res.raw) + } + return out +} + +// user registers a user and logs it in. +func (ts *tester) user(username, display string) account { + ts.t.Helper() + body := map[string]string{"username": username, "display_name": display, "password": testPassword} + res := ts.do(http.MethodPost, "/api/v1/users", "", body, nil) + ts.expect(res, http.StatusCreated, "") + registered := decode[user.Response](ts.t, res) + return account{Response: registered, token: ts.login(username, testPassword).Token} +} + +func basic(username, password string) map[string]string { + raw := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + return map[string]string{"Authorization": "Basic " + raw} +} + +func (ts *tester) login(username, password string) sessionBody { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic(username, password)) + ts.expect(res, http.StatusCreated, "") + return decode[sessionBody](ts.t, res) +} + +func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats", token, map[string]any{"title": title, "member_ids": members}, nil) + ts.expect(res, http.StatusCreated, "") + return decode[chat.ChatResponse](ts.t, res) +} + +func (ts *tester) message(token, chatID, text string) chat.MessageResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, nil) + ts.expect(res, http.StatusCreated, "") + return decode[chat.MessageResponse](ts.t, res) +} + +func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.ChatResponse]](ts.t, res) +} + +func (ts *tester) history(token, chatID, query string) pagination.PageResponse[chat.MessageResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats/"+chatID+"/messages"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.MessageResponse]](ts.t, res) +} + +func cursorQuery(limit int, cursor *string) string { + q := fmt.Sprintf("?limit=%d", limit) + if cursor != nil { + q += "&cursor=" + url.QueryEscape(*cursor) + } + return q +} + +func TestHealthEndpoints(t *testing.T) { + ts := newTester(t) + for _, path := range []string{"/health/live", "/health/ready"} { + res := ts.do(http.MethodGet, path, "", nil, nil) + ts.expect(res, http.StatusOK, "") + if strings.TrimSpace(string(res.raw)) != `{"status":"ok"}` { + t.Fatalf("%s body = %s", path, res.raw) + } + } + ts.a.Health.SetReady(false) + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") +} + +func TestRegisterUserShapeAndConflicts(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + if alice.ID == "" || alice.Username != "alice" || alice.DisplayName != "Алиса" { + t.Fatalf("user = %+v", alice) + } + if !strings.HasSuffix(alice.CreatedAt, "Z") || len(alice.CreatedAt) != len("2026-09-01T12:00:00.000Z") { + t.Fatalf("created_at = %q", alice.CreatedAt) + } + post := func(body any) response { return ts.do(http.MethodPost, "/api/v1/users", "", body, nil) } + ts.expect(post(map[string]string{"username": "alice", "display_name": "Other", "password": testPassword}), http.StatusConflict, "conflict") + ts.expect(post(map[string]string{"username": "bob", "display_name": "Bob"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "Alice", "display_name": "x"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "bob", "display_name": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": "bob", "display_name": "x", "password": "no"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(`{"username":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": 5, "display_name": "x"}), http.StatusBadRequest, "invalid_request") +} + +func TestCurrentUserIsTheTokenOwner(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + ts.user("bob", "Боб") + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got != alice.Response { + t.Fatalf("me = %+v, want %+v", got, alice.Response) + } +} + +func TestChatCreationRules(t *testing.T) { + ts := newTester(t) + alice, bob := ts.user("alice", "A"), ts.user("bob", "B") + created := ts.chat(alice.token, "Проект по ВНП", alice.ID, bob.ID) + if created.Title != "Проект по ВНП" || len(created.MemberIDs) != 2 || created.MemberIDs[0] != alice.ID || created.MemberIDs[1] != bob.ID { + t.Fatalf("chat = %+v", created) + } + unknown := "22222222-2222-4222-8222-222222222222" + invalid := map[string]any{ + "creator missing": map[string]any{"title": "t", "member_ids": []string{bob.ID}}, + "duplicate": map[string]any{"title": "t", "member_ids": []string{alice.ID, alice.ID}}, + "unknown member": map[string]any{"title": "t", "member_ids": []string{alice.ID, unknown}}, + "malformed uuid": map[string]any{"title": "t", "member_ids": []string{alice.ID, "not-a-uuid"}}, + "empty members": map[string]any{"title": "t", "member_ids": []string{}}, + "blank title": map[string]any{"title": "", "member_ids": []string{alice.ID}}, + "long title": map[string]any{"title": strings.Repeat("я", 101), "member_ids": []string{alice.ID}}, + "missing members": map[string]any{"title": "t"}, + "extra field": map[string]any{"title": "t", "member_ids": []string{alice.ID}, "owner": alice.ID}, + } + for name, body := range invalid { + res := ts.do(http.MethodPost, "/api/v1/chats", alice.token, body, nil) + if res.status != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body %s", name, res.status, res.raw) + } + } + ts.expect(ts.do(http.MethodPost, "/api/v1/chats", "", map[string]any{"title": "t", "member_ids": []string{alice.ID}}, nil), http.StatusUnauthorized, "unauthorized") +} + +func TestChatVisibilityAndListing(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + created := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + created.ID + + res := ts.do(http.MethodGet, path, bob.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[chat.ChatResponse](t, res); got.ID != created.ID || len(got.MemberIDs) != 2 { + t.Fatalf("GET chat = %+v", got) + } + ts.expect(ts.do(http.MethodGet, path, eve.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/33333333-3333-4333-8333-333333333333", alice.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/not-a-uuid", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, path, "", nil, nil), http.StatusUnauthorized, "unauthorized") + + listed := ts.chats(bob.token, "?limit=100") + if len(listed.Items) != 1 || listed.Items[0].ID != created.ID || listed.NextCursor != nil { + t.Fatalf("bob list = %+v", listed) + } + hidden := ts.do(http.MethodGet, "/api/v1/chats?limit=100", eve.token, nil, nil) + ts.expect(hidden, http.StatusOK, "") + if !strings.Contains(string(hidden.raw), `"items":[]`) || !strings.Contains(string(hidden.raw), `"next_cursor":null`) { + t.Fatalf("empty list must serialize items as [] and next_cursor as null: %s", hidden.raw) + } +} + +func TestChatListPagination(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + var expected []string + for i := 0; i < 5; i++ { + expected = append(expected, ts.chat(alice.token, fmt.Sprintf("chat %d", i), alice.ID).ID) + } + var walked []string + var cursor *string + for pages := 0; pages < 10; pages++ { + page := ts.chats(alice.token, cursorQuery(2, cursor)) + for _, c := range page.Items { + walked = append(walked, c.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walked %v, want %v", walked, expected) + } + + bob := ts.user("bob", "B") + foreign := ts.chats(alice.token, "?limit=1").NextCursor + ts.expect(ts.do(http.MethodGet, "/api/v1/chats"+cursorQuery(50, foreign), bob.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?cursor=", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?limit=abc", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessagesEndToEnd(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + + first := ts.message(alice.token, room.ID, "Привет, мир 👋") + second := ts.message(bob.token, room.ID, "Сообщение Боба") + if first.SenderID != alice.ID || second.SenderID != bob.ID { + t.Fatalf("sender must come from the bearer token: %+v %+v", first, second) + } + if first.ChatID != room.ID || first.Text != "Привет, мир 👋" { + t.Fatalf("message = %+v", first) + } + + page1 := ts.history(bob.token, room.ID, "?limit=1") + if len(page1.Items) != 1 || page1.NextCursor == nil || page1.Items[0] != first { + t.Fatalf("page1 = %+v", page1) + } + page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor)) + if len(page2.Items) != 1 || page2.NextCursor != nil || page2.Items[0] != second { + t.Fatalf("page2 = %+v", page2) + } + + post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, nil) } + ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": "x", "sender_id": bob.ID}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, `{"text":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(eve.token, map[string]string{"text": "чужое"}), http.StatusNotFound, "not_found") + ts.expect(post(eve.token, map[string]string{"text": ""}), http.StatusNotFound, "not_found") + ts.expect(post("", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4000)}), http.StatusCreated, "") + + get := func(actor, query string) response { return ts.do(http.MethodGet, path+query, actor, nil, nil) } + ts.expect(get(eve.token, ""), http.StatusNotFound, "not_found") + ts.expect(get(alice.token, "?limit=0"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?limit=101"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?cursor=garbage"), http.StatusBadRequest, "invalid_request") + + other := ts.chat(alice.token, "другой", alice.ID) + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+other.ID+"/messages"+cursorQuery(50, page1.NextCursor), alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessageHistoryWalkAcrossPages(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "t", alice.ID) + var expected []string + for i := 0; i < 23; i++ { + expected = append(expected, ts.message(alice.token, room.ID, fmt.Sprintf("m%d", i)).ID) + } + var walked []string + var cursor *string + for { + page := ts.history(alice.token, room.ID, cursorQuery(5, cursor)) + for _, m := range page.Items { + walked = append(walked, m.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walk mismatch:\n%v\n%v", walked, expected) + } +} + +func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) { + ts := newTester(t) + const clients, perClient = 10, 10 + clientsList := make([]account, 0, clients) + members := make([]string, 0, clients) + for i := 0; i < clients; i++ { + c := ts.user(fmt.Sprintf("client_%d", i), "C") + clientsList = append(clientsList, c) + members = append(members, c.ID) + } + room := ts.chat(clientsList[0].token, "нагрузка", members...) + + var wg sync.WaitGroup + for n, c := range clientsList { + wg.Go(func() { + for i := 0; i < perClient; i++ { + res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, nil) + if res.status != http.StatusCreated { + t.Errorf("client %d: status %d body %s", n, res.status, res.raw) + } + } + }) + } + wg.Wait() + + page := ts.history(clientsList[0].token, room.ID, "?limit=100") + if len(page.Items) != clients*perClient || page.NextCursor != nil { + t.Fatalf("got %d messages, next=%v; want %d and null", len(page.Items), page.NextCursor, clients*perClient) + } + ids := make([]string, 0, len(page.Items)) + unique := make(map[string]struct{}, len(page.Items)) + for _, m := range page.Items { + ids = append(ids, m.ID) + unique[m.ID] = struct{}{} + } + if !sort.StringsAreSorted(ids) { + t.Fatal("history is not ordered by id") + } + if len(unique) != clients*perClient { + t.Fatalf("%d unique ids, want %d", len(unique), clients*perClient) + } +} + +func TestUnknownRouteAndMethodAreContractErrors(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodGet, "/api/v1/nothing", "", nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodDelete, "/api/v1/users", "", nil, nil), http.StatusMethodNotAllowed, "invalid_request") + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", "{}", map[string]string{"Content-Type": "text/plain"}), http.StatusUnsupportedMediaType, "unsupported_media_type") + big := map[string]string{"username": "big", "display_name": strings.Repeat("x", 70*1024)} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", big, nil), http.StatusRequestEntityTooLarge, "payload_too_large") +} diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go new file mode 100644 index 00000000..602d1e2a --- /dev/null +++ b/internal/app/auth_test.go @@ -0,0 +1,162 @@ +package app_test + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "messenger/internal/user" +) + +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +func TestLoginReturnsBearerSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + before := time.Now() + + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)) + ts.expect(res, http.StatusCreated, "") + s := decode[sessionBody](t, res) + + if s.TokenType != "Bearer" || len(s.Token) != 43 || s.Token == alice.token { + t.Fatalf("session = %+v; want a fresh 43-character Bearer token", s) + } + if s.User != alice.Response { + t.Fatalf("user = %+v, want %+v", s.User, alice.Response) + } + expires, err := time.Parse(time.RFC3339, s.ExpiresAt) + if err != nil || !strings.HasSuffix(s.ExpiresAt, "Z") { + t.Fatalf("expires_at = %q: %v", s.ExpiresAt, err) + } + if ttl := expires.Sub(before); ttl < 59*time.Minute || ttl > 61*time.Minute { + t.Fatalf("expires_at is %v after login, want the 1h TTL", ttl) + } + if strings.Contains(string(res.raw), "password") || strings.Contains(string(res.raw), "argon2") { + t.Fatalf("session response leaks password data: %s", res.raw) + } +} + +func TestLoginFailuresAreUniformBasicChallenges(t *testing.T) { + ts := newTester(t) + ts.user("alice", "Алиса") + encoded := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) } + cases := map[string]map[string]string{ + "wrong password": basic("alice", testPassword+"!"), + "unknown user": basic("mallory", testPassword), + "no header": nil, + "bearer scheme": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "not base64": {"Authorization": "Basic !!!"}, + "no colon": {"Authorization": "Basic " + encoded("alice")}, + "nul username": basic("a\x00b", testPassword), + "non-utf-8 name": basic("\xff\xfe", testPassword), + } + messages := map[string]string{} + for name, headers := range cases { + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != basicChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, basicChallenge) + } + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + messages[name] = eb.Error.Message + } + if messages["wrong password"] != messages["unknown user"] { + t.Fatalf("wrong password and unknown user must look the same: %q vs %q", messages["wrong password"], messages["unknown user"]) + } +} + +// The password is UTF-8 and may contain colons; only the first colon of the +// Basic credentials separates the username (contracts/README.md). +func TestLoginAcceptsColonAndUnicodePasswords(t *testing.T) { + ts := newTester(t) + const password = "пароль:с:двоеточием 👋" + body := map[string]string{"username": "alice", "display_name": "Алиса", "password": password} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") + s := ts.login("alice", password) + if s.User.Username != "alice" { + t.Fatalf("logged in as %+v", s.User) + } +} + +func TestProtectedRoutesAcceptOnlyBearer(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + me := func(headers map[string]string) response { + return ts.do(http.MethodGet, "/api/v1/users/me", "", nil, headers) + } + rejected := map[string]map[string]string{ + "no header": nil, + "basic with valid credentials": basic("alice", testPassword), + "x-user-id alone": {"X-User-Id": alice.ID}, + "unknown token": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "empty token": {"Authorization": "Bearer "}, + "scheme only": {"Authorization": "Bearer"}, + "other scheme": {"Authorization": "Token " + alice.token}, + "two tokens": {"Authorization": "Bearer " + alice.token + " " + eve.token}, + } + for name, headers := range rejected { + res := me(headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != bearerChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, bearerChallenge) + } + } + accepted := map[string]map[string]string{ + "lowercase scheme": {"Authorization": "bearer " + alice.token}, + "several spaces": {"Authorization": "Bearer " + alice.token}, + "x-user-id is ignored": {"Authorization": "Bearer " + alice.token, "X-User-Id": eve.ID}, + } + for name, headers := range accepted { + res := me(headers) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got.ID != alice.ID { + t.Errorf("%s: caller = %s, want alice", name, got.ID) + } + } +} + +func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + other := ts.login("alice", testPassword) + + res := ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil) + ts.expect(res, http.StatusNoContent, "") + if len(res.raw) != 0 { + t.Fatalf("204 must have no body, got %q", res.raw) + } + + for _, again := range []response{ + ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), + ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), + } { + ts.expect(again, http.StatusUnauthorized, "unauthorized") + if again.challenge != bearerChallenge { + t.Fatalf("revoked token: WWW-Authenticate = %q, want %q", again.challenge, bearerChallenge) + } + } + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") +} + +func TestSessionExpiresAfterTTL(t *testing.T) { + ts := newTesterWithTTL(t, time.Second) + alice := ts.user("alice", "A") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") + time.Sleep(1200 * time.Millisecond) + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized") +} + +func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) { + ts := newTester(t) + body := map[string]string{"username": "alice", "display_name": "A", "password": testPassword} + headers := map[string]string{"Authorization": "Bearer garbage"} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, headers), http.StatusCreated, "") +} diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go new file mode 100644 index 00000000..cadc4ee6 --- /dev/null +++ b/internal/app/failure_test.go @@ -0,0 +1,100 @@ +package app_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +// newIsolatedTester assembles the application over containers of its own, +// so a test can stop or pause them without disturbing the shared ones. +func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *testenv.DB, *testenv.KV) { + t.Helper() + ctx := context.Background() + pg, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pg.Close) + vk, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(vk.Close) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{ + Log: log, + DB: pg.Pool, + Sessions: vk.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + RequestTimeout: requestTimeout, + }) + return &tester{t: t, a: a}, pg, vk +} + +// Without the session store no caller can be authenticated: the API answers +// 503, never 401 (the token is not at fault) and never as an anonymous user. +// Registration does not need the store and keeps working. +func TestValkeyOutageAnswers503(t *testing.T) { + ts, _, vk := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + room := ts.chat(alice.token, "t", alice.ID) + + if err := vk.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+room.ID, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") + + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") +} + +func TestPostgresOutageAnswers503(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + + if err := pg.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/chats", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") +} + +// A database that accepts connections but never answers must not hold the +// request: the request deadline turns it into a 503. +func TestHungPostgresAnswers503WithinTheRequestTimeout(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 300*time.Millisecond) + alice := ts.user("alice", "Алиса") + + ctx := context.Background() + if err := pg.Pause(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.Unpause(ctx) }) + + for _, path := range []string{"/api/v1/chats", "/health/ready"} { + start := time.Now() + ts.expect(ts.do(http.MethodGet, path, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("%s took %v with a hung database, want it bounded", path, elapsed) + } + } +} diff --git a/internal/auth/handler.go b/internal/auth/handler.go new file mode 100644 index 00000000..e8c8ccef --- /dev/null +++ b/internal/auth/handler.go @@ -0,0 +1,89 @@ +package auth + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/user" +) + +// Challenges sent with every 401: Basic on the login endpoint, Bearer on +// every protected one. +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +var errCredentialsRequired = apperr.Unauthorized("basic credentials are required") + +type Handler struct { + sessions *Service +} + +func NewHandler(sessions *Service) *Handler { + return &Handler{sessions: sessions} +} + +// Register mounts login on the public router and logout on the router +// guarded by RequireSession. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/auth/sessions", h.login) + protected.DELETE("/auth/sessions/current", h.logout) +} + +type sessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +// login accepts only the Authorization header; the body is not read. +// BasicAuth splits at the first colon, so a password may contain colons. +func (h *Handler) login(c *gin.Context) { + username, password, ok := c.Request.BasicAuth() + if !ok { + unauthorized(c, basicChallenge, errCredentialsRequired) + return + } + issued, err := h.sessions.Login(c.Request.Context(), username, password) + if err != nil { + unauthorized(c, basicChallenge, err) + return + } + httpx.JSON(c, http.StatusCreated, sessionResponse{ + Token: issued.Token, + TokenType: "Bearer", + ExpiresAt: httpx.Timestamp(issued.ExpiresAt), + User: user.ToResponse(issued.User), + }) +} + +func (h *Handler) logout(c *gin.Context) { + token, ok := c.Get(keyToken) + if !ok { + httpx.Error(c, errors.New("logout route is missing the RequireSession middleware")) + return + } + if err := h.sessions.Logout(c.Request.Context(), token.(string)); err != nil { + httpx.Error(c, err) + return + } + // Status alone is buffered by gin; writing the header now sends the 204 + // with an empty body. + c.Status(http.StatusNoContent) + c.Writer.WriteHeaderNow() +} + +// unauthorized adds the challenge only to a 401: an unavailable store is a +// 503 and must not invite the client to retry its credentials. +func unauthorized(c *gin.Context, challenge string, err error) { + if apperr.KindOf(err) == apperr.KindUnauthorized { + c.Header("WWW-Authenticate", challenge) + } + httpx.Error(c, err) +} diff --git a/internal/auth/main_test.go b/internal/auth/main_test.go new file mode 100644 index 00000000..a542ee95 --- /dev/null +++ b/internal/auth/main_test.go @@ -0,0 +1,33 @@ +package auth_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go new file mode 100644 index 00000000..13dbeeae --- /dev/null +++ b/internal/auth/middleware.go @@ -0,0 +1,48 @@ +package auth + +import ( + "strings" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/httpx" +) + +const keyToken = "messenger.session_token" + +// RequireSession admits a request only with Authorization: Bearer of +// a live session and records its owner for the handlers behind it. Any other +// identification, including Basic credentials and X-User-Id, is ignored, so +// it can neither grant access nor change the caller. +func RequireSession(sessions *Service) gin.HandlerFunc { + return func(c *gin.Context) { + token, ok := bearerToken(c.GetHeader("Authorization")) + if !ok { + unauthorized(c, bearerChallenge, ErrUnauthenticated) + return + } + userID, err := sessions.Authenticate(c.Request.Context(), token) + if err != nil { + unauthorized(c, bearerChallenge, err) + return + } + httpx.SetUserID(c, userID) + c.Set(keyToken, token) + c.Next() + } +} + +// bearerToken parses "Bearer ": the scheme is case-insensitive and +// may be followed by several spaces (RFC 7235); the token itself never +// contains whitespace. +func bearerToken(header string) (string, bool) { + scheme, rest, found := strings.Cut(header, " ") + if !found || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token := strings.TrimLeft(rest, " ") + if token == "" || strings.ContainsAny(token, " \t") { + return "", false + } + return token, true +} diff --git a/internal/auth/repository.go b/internal/auth/repository.go new file mode 100644 index 00000000..4a168f59 --- /dev/null +++ b/internal/auth/repository.go @@ -0,0 +1,18 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores the session until its absolute ExpiresAt. + Create(ctx context.Context, s Session) error + // UserID returns the owner of a live session, or ErrUnauthenticated when + // the token is unknown, expired or revoked: the store cannot tell these + // apart, and the client must not either. + UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) + // Delete revokes the session; deleting a missing one is not an error. + Delete(ctx context.Context, tokenHash string) error +} diff --git a/internal/auth/service.go b/internal/auth/service.go new file mode 100644 index 00000000..458a4744 --- /dev/null +++ b/internal/auth/service.go @@ -0,0 +1,64 @@ +package auth + +import ( + "context" + "time" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/user" +) + +var ErrUnauthenticated = apperr.Unauthorized("a valid bearer token is required") + +// Credentials is what this module needs from the users module: the user +// whose password matches, or user.ErrInvalidCredentials. +type Credentials interface { + Authenticate(ctx context.Context, username, password string) (user.User, error) +} + +// Issued is a new session as the client sees it; Token is shown only here. +type Issued struct { + Token string + ExpiresAt time.Time + User user.User +} + +type Service struct { + users Credentials + sessions Repository + ttl time.Duration +} + +func NewService(users Credentials, sessions Repository, ttl time.Duration) *Service { + return &Service{users: users, sessions: sessions, ttl: ttl} +} + +// Login checks the credentials first, so a store outage never hands out a +// token without a stored session. The expiry is fixed here and never +// extended; it is truncated to milliseconds, the precision of both the API +// timestamps and the store. +func (s *Service) Login(ctx context.Context, username, password string) (Issued, error) { + u, err := s.users.Authenticate(ctx, username, password) + if err != nil { + return Issued{}, err + } + token := newToken() + expiresAt := time.Now().Add(s.ttl).Truncate(time.Millisecond).UTC() + if err := s.sessions.Create(ctx, Session{TokenHash: HashToken(token), UserID: u.ID, ExpiresAt: expiresAt}); err != nil { + return Issued{}, err + } + return Issued{Token: token, ExpiresAt: expiresAt, User: u}, nil +} + +// Authenticate returns the owner of the token's live session or +// ErrUnauthenticated. A store failure stays an unavailability, never +// ErrUnauthenticated: an outage must not look like a bad token to the client. +func (s *Service) Authenticate(ctx context.Context, token string) (uuid.UUID, error) { + return s.sessions.UserID(ctx, HashToken(token)) +} + +func (s *Service) Logout(ctx context.Context, token string) error { + return s.sessions.Delete(ctx, HashToken(token)) +} diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go new file mode 100644 index 00000000..fb5d767b --- /dev/null +++ b/internal/auth/service_test.go @@ -0,0 +1,145 @@ +package auth_test + +import ( + "context" + "encoding/base64" + "errors" + "strings" + "testing" + "time" + + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/platform/apperr" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +type fixture struct { + ctx context.Context + svc *auth.Service + alice user.User +} + +// newFixture wires the real users service and session store over empty +// databases and registers alice. +func newFixture(t *testing.T, ttl time.Duration) fixture { + t.Helper() + db.Reset(t) + kv.Reset(t) + ctx := context.Background() + users := user.NewService(userstore.NewUsers(db.Pool)) + alice, err := users.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatal(err) + } + return fixture{ctx: ctx, svc: auth.NewService(users, authstore.NewSessions(kv.Client), ttl), alice: alice} +} + +func TestLoginIssuesOpaqueTokenBackedByStoredSession(t *testing.T) { + f := newFixture(t, time.Hour) + before := time.Now() + issued, err := f.svc.Login(f.ctx, "alice", pw) + after := time.Now() + if err != nil { + t.Fatalf("Login: %v", err) + } + raw, err := base64.RawURLEncoding.DecodeString(issued.Token) + if err != nil || len(raw) != 32 { + t.Fatalf("token %q must be 32 random bytes in base64url, got %d bytes, %v", issued.Token, len(raw), err) + } + if strings.Contains(issued.Token, f.alice.ID.String()) || strings.Contains(issued.Token, "alice") { + t.Fatal("token must not carry the identity") + } + if issued.User != f.alice { + t.Fatalf("User = %+v, want alice", issued.User) + } + if issued.ExpiresAt.Before(before.Add(time.Hour).Add(-time.Millisecond)) || issued.ExpiresAt.After(after.Add(time.Hour)) { + t.Fatalf("ExpiresAt = %v, want login time + 1h", issued.ExpiresAt) + } + + key := "session:" + auth.HashToken(issued.Token) + owner, err := kv.Client.Get(f.ctx, key).Result() + if err != nil || owner != f.alice.ID.String() { + t.Fatalf("stored owner = %q, %v; want alice", owner, err) + } + expireAt, err := kv.Client.PExpireTime(f.ctx, key).Result() + if err != nil || expireAt != time.Duration(issued.ExpiresAt.UnixMilli())*time.Millisecond { + t.Fatalf("store expiry = %v, %v; want the absolute ExpiresAt %d ms", expireAt, err, issued.ExpiresAt.UnixMilli()) + } + keys, err := kv.Client.Keys(f.ctx, "*").Result() + if err != nil { + t.Fatal(err) + } + for _, k := range keys { + if strings.Contains(k, issued.Token) { + t.Fatal("the raw token must never be stored") + } + } +} + +func TestLoginWithBadCredentialsCreatesNoSession(t *testing.T) { + f := newFixture(t, time.Hour) + for name, in := range map[string][2]string{ + "wrong password": {"alice", pw + "!"}, + "unknown user": {"mallory", pw}, + } { + if _, err := f.svc.Login(f.ctx, in[0], in[1]); !errors.Is(err, user.ErrInvalidCredentials) { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if n, err := kv.Client.DBSize(f.ctx).Result(); err != nil || n != 0 { + t.Fatalf("sessions = %d, %v; want none", n, err) + } +} + +func TestAuthenticateResolvesTheOwner(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if first.Token == second.Token { + t.Fatal("every login must get its own token") + } + for _, token := range []string{first.Token, second.Token} { + id, err := f.svc.Authenticate(f.ctx, token) + if err != nil || id != f.alice.ID { + t.Fatalf("Authenticate = %v, %v; want alice", id, err) + } + } + _, err := f.svc.Authenticate(f.ctx, "unknown-token") + if !errors.Is(err, auth.ErrUnauthenticated) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Fatalf("unknown token: err = %v, want ErrUnauthenticated", err) + } +} + +func TestLogoutRevokesOnlyThatSession(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if err := f.svc.Logout(f.ctx, first.Token); err != nil { + t.Fatalf("Logout: %v", err) + } + if _, err := f.svc.Authenticate(f.ctx, first.Token); !errors.Is(err, auth.ErrUnauthenticated) { + t.Fatalf("revoked token: err = %v, want ErrUnauthenticated", err) + } + if id, err := f.svc.Authenticate(f.ctx, second.Token); err != nil || id != f.alice.ID { + t.Fatalf("other session: %v, %v; want still valid", id, err) + } +} + +func TestSessionExpiresAfterTTL(t *testing.T) { + f := newFixture(t, time.Second) + issued, err := f.svc.Login(f.ctx, "alice", pw) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil { + t.Fatalf("fresh session: %v", err) + } + time.Sleep(time.Until(issued.ExpiresAt) + 100*time.Millisecond) + if _, err := f.svc.Authenticate(f.ctx, issued.Token); !errors.Is(err, auth.ErrUnauthenticated) { + t.Fatalf("expired session: err = %v, want ErrUnauthenticated", err) + } +} diff --git a/internal/auth/session.go b/internal/auth/session.go new file mode 100644 index 00000000..daedb116 --- /dev/null +++ b/internal/auth/session.go @@ -0,0 +1,39 @@ +// Package auth authenticates callers. +// +// A login with Basic credentials creates a server-side session and returns an +// opaque bearer token; every protected request presents the token, and logout +// deletes the session, which revokes the token at once. The store keeps only +// the SHA-256 of a token: a token is a random 256-bit value, so a fast digest +// cannot be reversed by guessing, and a leaked store yields no usable tokens. +// Passwords need the opposite treatment (a slow KDF), see platform/password. +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "time" + + "github.com/google/uuid" +) + +type Session struct { + TokenHash string + UserID uuid.UUID + ExpiresAt time.Time +} + +// newToken returns 32 random bytes in base64url: 43 characters, 256 bits. +func newToken() string { + raw := make([]byte, 32) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(raw) + return base64.RawURLEncoding.EncodeToString(raw) +} + +// HashToken is the key under which a token's session is stored. +func HashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/auth/store/valkey.go b/internal/auth/store/valkey.go new file mode 100644 index 00000000..ad7a36e1 --- /dev/null +++ b/internal/auth/store/valkey.go @@ -0,0 +1,64 @@ +// Package store holds the session store implementations. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "messenger/internal/auth" + "messenger/internal/platform/apperr" +) + +const keyPrefix = "session:" + +// Sessions keeps one key per session, session:, holding the +// owner's id. Valkey expires the key itself, so no cleanup job is needed. +type Sessions struct { + client *redis.Client +} + +var _ auth.Repository = (*Sessions)(nil) + +func NewSessions(client *redis.Client) *Sessions { + return &Sessions{client: client} +} + +// Create sets an absolute expiry (EXAT/PXAT): a restart of the store replays +// it from the append-only file and does not extend the session. +func (r *Sessions) Create(ctx context.Context, s auth.Session) error { + err := r.client.SetArgs(ctx, keyPrefix+s.TokenHash, s.UserID.String(), redis.SetArgs{ExpireAt: s.ExpiresAt}).Err() + if err != nil { + return unavailable(err) + } + return nil +} + +func (r *Sessions) UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) { + raw, err := r.client.Get(ctx, keyPrefix+tokenHash).Result() + if errors.Is(err, redis.Nil) { + return uuid.Nil, auth.ErrUnauthenticated + } + if err != nil { + return uuid.Nil, unavailable(err) + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, fmt.Errorf("session holds a malformed user id: %w", err) + } + return id, nil +} + +func (r *Sessions) Delete(ctx context.Context, tokenHash string) error { + if err := r.client.Del(ctx, keyPrefix+tokenHash).Err(); err != nil { + return unavailable(err) + } + return nil +} + +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "session store unavailable", err) +} diff --git a/internal/chat/chat.go b/internal/chat/chat.go new file mode 100644 index 00000000..c9281351 --- /dev/null +++ b/internal/chat/chat.go @@ -0,0 +1,82 @@ +// Package chat is the chats module. Entities mirror table rows and +// carry no behaviour; membership is a query rather than a field, and +// ChatWithMembers is the projection the API renders. Both entities derive +// CreatedAt from their UUIDv7 id, so the history order (created_at, id) is +// plain id order. +package chat + +import ( + "bytes" + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxTitleCodePoints = 100 + MaxMembers = 100 +) + +// ErrNotFound covers both a missing chat and a chat the caller is not a +// member of, so outsiders cannot probe for existence. +var ErrNotFound = apperr.NotFound("chat not found") + +type Chat struct { + ID uuid.UUID + Title string + CreatedAt time.Time +} + +type ChatWithMembers struct { + Chat + MemberIDs []uuid.UUID +} + +func NewChat(id uuid.UUID, title string) (Chat, error) { + if strings.TrimSpace(title) == "" { + return Chat{}, apperr.Invalid("title must not be blank") + } + if utf8.RuneCountInString(title) > MaxTitleCodePoints { + return Chat{}, apperr.Invalid(fmt.Sprintf("title must be at most %d characters", MaxTitleCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(title, 0) { + return Chat{}, apperr.Invalid("title must not contain the NUL character") + } + return Chat{ID: id, Title: title, CreatedAt: ident.TimeOf(id)}, nil +} + +// ValidateMembers checks the member list of a new chat and returns its own +// copy ordered by id: members are a set, and this is the one order in which +// every read returns them. +func ValidateMembers(memberIDs []uuid.UUID, creator uuid.UUID) ([]uuid.UUID, error) { + if len(memberIDs) == 0 { + return nil, apperr.Invalid("member_ids must not be empty") + } + if len(memberIDs) > MaxMembers { + return nil, apperr.Invalid(fmt.Sprintf("member_ids must have at most %d entries", MaxMembers)) + } + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + for _, m := range memberIDs { + if m == uuid.Nil { + return nil, apperr.Invalid("member_ids must not contain the nil uuid") + } + if _, dup := seen[m]; dup { + return nil, apperr.Invalid("member_ids must be unique") + } + seen[m] = struct{}{} + } + if _, ok := seen[creator]; !ok { + return nil, apperr.Invalid("member_ids must include the creator") + } + members := slices.Clone(memberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { return bytes.Compare(a[:], b[:]) }) + return members, nil +} diff --git a/internal/chat/chat_test.go b/internal/chat/chat_test.go new file mode 100644 index 00000000..f63f60e3 --- /dev/null +++ b/internal/chat/chat_test.go @@ -0,0 +1,88 @@ +package chat_test + +import ( + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewChatDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + c, err := chat.NewChat(id, "Проект") + if err != nil { + t.Fatalf("NewChat error: %v", err) + } + if c.ID != id || c.Title != "Проект" || !c.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("unexpected chat %+v", c) + } +} + +func TestNewChatValidatesTitle(t *testing.T) { + // PostgreSQL text cannot hold U+0000, so a title with it is invalid input, + // not a storage failure. + for _, title := range []string{"", " ", strings.Repeat("я", 101), "nul\x00inside"} { + _, err := chat.NewChat(ident.New(), title) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("title %q must be rejected as invalid, got %v", title, err) + } + } + if _, err := chat.NewChat(ident.New(), strings.Repeat("я", 100)); err != nil { + t.Errorf("100 code points must be accepted: %v", err) + } +} + +// Members come back in one canonical order (by id) whatever order the client +// sent, so the created chat and every later read of it agree. +func TestValidateMembersCopiesInCanonicalOrder(t *testing.T) { + alice, bob := ident.New(), ident.New() + input := []uuid.UUID{bob, alice} + members, err := chat.ValidateMembers(input, alice) + if err != nil { + t.Fatalf("ValidateMembers error: %v", err) + } + if len(members) != 2 || members[0] != alice || members[1] != bob { + t.Fatalf("members must be ordered by id, got %v", members) + } + input[1] = ident.New() + if members[0] != alice { + t.Fatal("result must not alias the input slice") + } +} + +func TestValidateMembersRejectsInvalidLists(t *testing.T) { + alice, bob := ident.New(), ident.New() + cases := map[string][]uuid.UUID{ + "empty": {}, + "nil": nil, + "missing creator": {bob}, + "duplicate": {alice, bob, bob}, + "duplicate self": {alice, alice}, + "too many": append([]uuid.UUID{alice}, many(100)...), + "nil uuid member": {alice, uuid.Nil}, + } + for name, members := range cases { + _, err := chat.ValidateMembers(members, alice) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: want invalid, got %v", name, err) + } + if err != nil && !strings.Contains(err.Error(), "member_ids") { + t.Errorf("%s: message must name the field: %v", name, err) + } + } + if _, err := chat.ValidateMembers(append([]uuid.UUID{alice}, many(99)...), alice); err != nil { + t.Errorf("100 members must be accepted: %v", err) + } +} + +func many(n int) []uuid.UUID { + out := make([]uuid.UUID, n) + for i := range out { + out[i] = ident.New() + } + return out +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go new file mode 100644 index 00000000..9a2c2cc9 --- /dev/null +++ b/internal/chat/handler.go @@ -0,0 +1,192 @@ +package chat + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +type Handler struct { + chats *Service +} + +func NewHandler(chats *Service) *Handler { + return &Handler{chats: chats} +} + +// Register mounts the chat endpoints on a router guarded by the authentication middleware. +func (h *Handler) Register(protected gin.IRouter) { + protected.POST("/chats", httpx.WithUser(h.create)) + protected.GET("/chats", httpx.WithUser(h.list)) + protected.GET("/chats/:chat_id", httpx.WithUser(h.get)) + protected.POST("/chats/:chat_id/messages", httpx.WithUser(h.send)) + protected.GET("/chats/:chat_id/messages", httpx.WithUser(h.history)) +} + +type createChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type createMessageRequest struct { + Text string `json:"text"` +} + +type ChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type MessageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` +} + +func toChatResponse(c ChatWithMembers) ChatResponse { + members := make([]string, 0, len(c.MemberIDs)) + for _, m := range c.MemberIDs { + members = append(members, m.String()) + } + return ChatResponse{ + ID: c.ID.String(), + Title: c.Title, + MemberIDs: members, + CreatedAt: httpx.Timestamp(c.CreatedAt), + } +} + +func toMessageResponse(m Message) MessageResponse { + return MessageResponse{ + ID: m.ID.String(), + ChatID: m.ChatID.String(), + SenderID: m.SenderID.String(), + Text: m.Text, + CreatedAt: httpx.Timestamp(m.CreatedAt), + } +} + +func (h *Handler) create(c *gin.Context, actor uuid.UUID) { + var req createChatRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + members := make([]uuid.UUID, 0, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + id, err := ident.Parse(raw) + if err != nil { + httpx.Error(c, apperr.Invalid("member_ids must contain valid UUIDs")) + return + } + members = append(members, id) + } + chat, err := h.chats.Create(c.Request.Context(), actor, req.Title, members) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toChatResponse(chat)) +} + +func (h *Handler) list(c *gin.Context, actor uuid.UUID) { + limit, after, err := listing(c, actor) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.List(c.Request.Context(), actor, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, actor, toChatResponse)) +} + +func (h *Handler) get(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + chat, err := h.chats.Get(c.Request.Context(), actor, chatID) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toChatResponse(chat)) +} + +func (h *Handler) send(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + var req createMessageRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + message, err := h.chats.Send(c.Request.Context(), actor, chatID, req.Text) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toMessageResponse(message)) +} + +func (h *Handler) history(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + limit, after, err := listing(c, chatID) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.History(c.Request.Context(), actor, chatID, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse)) +} + +func chatParam(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("chat_id must be a UUID") + } + return id, nil +} + +// listing parses limit and cursor; the cursor must belong to scope, and a +// present but empty cursor is rejected like a malformed one. +func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) { + limit, err := pagination.ParseLimit(c.Query("limit")) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, err.Error(), err) + } + token, present := c.GetQuery("cursor") + if !present { + return limit, nil, nil + } + last, err := pagination.DecodeCursor(token, scope) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, "cursor is invalid for this resource", err) + } + return limit, &last, nil +} diff --git a/internal/chat/message.go b/internal/chat/message.go new file mode 100644 index 00000000..c28f6d7d --- /dev/null +++ b/internal/chat/message.go @@ -0,0 +1,44 @@ +package chat + +import ( + "fmt" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const MaxTextCodePoints = 4000 + +type Message struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +// NewMessage keeps the text verbatim; only blank or oversized text is rejected. +func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { + if strings.TrimSpace(text) == "" { + return Message{}, apperr.Invalid("text must not be blank") + } + if utf8.RuneCountInString(text) > MaxTextCodePoints { + return Message{}, apperr.Invalid(fmt.Sprintf("text must be at most %d characters", MaxTextCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(text, 0) { + return Message{}, apperr.Invalid("text must not contain the NUL character") + } + return Message{ + ID: id, + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: ident.TimeOf(id), + }, nil +} diff --git a/internal/chat/message_test.go b/internal/chat/message_test.go new file mode 100644 index 00000000..3cb76738 --- /dev/null +++ b/internal/chat/message_test.go @@ -0,0 +1,47 @@ +package chat_test + +import ( + "strings" + "testing" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewMessageDerivesCreatedAt(t *testing.T) { + id, chatID, alice := ident.New(), ident.New(), ident.New() + m, err := chat.NewMessage(id, chatID, alice, "Привет, мир 👋") + if err != nil { + t.Fatalf("NewMessage error: %v", err) + } + if m.ID != id || m.ChatID != chatID || m.SenderID != alice || m.Text != "Привет, мир 👋" { + t.Fatalf("unexpected message %+v", m) + } + if !m.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", m.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewMessageValidatesTextByCodePoints(t *testing.T) { + chatID, alice := ident.New(), ident.New() + if _, err := chat.NewMessage(ident.New(), chatID, alice, strings.Repeat("я", 4000)); err != nil { + t.Errorf("4000 code points must be accepted: %v", err) + } + for _, text := range []string{"", " ", "\n\t ", strings.Repeat("я", 4001), "nul\x00inside"} { + _, err := chat.NewMessage(ident.New(), chatID, alice, text) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("text %q must be rejected as invalid, got %v", text, err) + } + } +} + +func TestNewMessageKeepsSurroundingWhitespace(t *testing.T) { + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " hi ") + if err != nil { + t.Fatal(err) + } + if m.Text != " hi " { + t.Fatalf("text must be stored verbatim, got %q", m.Text) + } +} diff --git a/internal/chat/repository.go b/internal/chat/repository.go new file mode 100644 index 00000000..765b1840 --- /dev/null +++ b/internal/chat/repository.go @@ -0,0 +1,31 @@ +package chat + +import ( + "context" + + "github.com/google/uuid" +) + +// ChatRepository stores chats and their membership. Membership is answered +// by queries; the projection with members is assembled by the store in one +// query, never by loading a member list into the entity. Members are always +// returned ordered by id. +type ChatRepository interface { + // Create stores the chat row and its members atomically. + Create(ctx context.Context, chat Chat, memberIDs []uuid.UUID) error + // GetForMember returns the chat with members, or ErrNotFound when the + // chat does not exist or the user is not a member. + GetForMember(ctx context.Context, chatID, userID uuid.UUID) (ChatWithMembers, error) + IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) + // ListByMember returns up to limit chats of the user ordered by + // (created_at, id), strictly after the chat identified by after when set. + ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]ChatWithMembers, error) +} + +type MessageRepository interface { + Append(ctx context.Context, message Message) error + // ListByChat returns up to limit messages ordered by (created_at, id), + // strictly after the message identified by after when set. An unknown + // chat yields an empty history. + ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]Message, error) +} diff --git a/internal/chat/service.go b/internal/chat/service.go new file mode 100644 index 00000000..97e09248 --- /dev/null +++ b/internal/chat/service.go @@ -0,0 +1,101 @@ +package chat + +import ( + "context" + "fmt" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +// UserLookup is what this module needs from the users module: one batched +// existence check. It is declared here, on the consumer side, so this +// package never imports the users package and tests substitute a map. +type UserLookup interface { + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} + +type Service struct { + chats ChatRepository + messages MessageRepository + users UserLookup +} + +func NewService(chats ChatRepository, messages MessageRepository, users UserLookup) *Service { + return &Service{chats: chats, messages: messages, users: users} +} + +func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) { + chat, err := NewChat(ident.New(), title) + if err != nil { + return ChatWithMembers{}, err + } + members, err := ValidateMembers(memberIDs, creator) + if err != nil { + return ChatWithMembers{}, err + } + found, err := s.users.ExistingIDs(ctx, members) + if err != nil { + return ChatWithMembers{}, fmt.Errorf("check members: %w", err) + } + if len(found) != len(members) { + return ChatWithMembers{}, apperr.Invalid("member_ids must reference registered users") + } + if err := s.chats.Create(ctx, chat, members); err != nil { + return ChatWithMembers{}, err + } + return ChatWithMembers{Chat: chat, MemberIDs: members}, nil +} + +func (s *Service) Get(ctx context.Context, actor, chatID uuid.UUID) (ChatWithMembers, error) { + return s.chats.GetForMember(ctx, chatID, actor) +} + +func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[ChatWithMembers], error) { + items, err := s.chats.ListByMember(ctx, actor, after, limit+1) + if err != nil { + return pagination.Page[ChatWithMembers]{}, err + } + return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil +} + +func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string) (Message, error) { + if err := s.requireMember(ctx, actor, chatID); err != nil { + return Message{}, err + } + message, err := NewMessage(ident.New(), chatID, actor, text) + if err != nil { + return Message{}, err + } + if err := s.messages.Append(ctx, message); err != nil { + return Message{}, err + } + return message, nil +} + +func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) { + if err := s.requireMember(ctx, actor, chatID); err != nil { + return pagination.Page[Message]{}, err + } + items, err := s.messages.ListByChat(ctx, chatID, after, limit+1) + if err != nil { + return pagination.Page[Message]{}, err + } + return pagination.Cut(items, limit, func(m Message) uuid.UUID { return m.ID }), nil +} + +// requireMember runs before any field validation, so an outsider gets the +// same answer for a foreign chat as for a missing one. +func (s *Service) requireMember(ctx context.Context, actor, chatID uuid.UUID) error { + ok, err := s.chats.IsMember(ctx, chatID, actor) + if err != nil { + return fmt.Errorf("check membership: %w", err) + } + if !ok { + return ErrNotFound + } + return nil +} diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go new file mode 100644 index 00000000..7e9525d8 --- /dev/null +++ b/internal/chat/service_test.go @@ -0,0 +1,234 @@ +package chat_test + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync/atomic" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +// countingUsers passes through to the real users service and counts the +// lookups, which proves that member checks are batched. +type countingUsers struct { + *user.Service + calls atomic.Int32 +} + +func (c *countingUsers) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + c.calls.Add(1) + return c.Service.ExistingIDs(ctx, ids) +} + +type fixture struct { + svc *chat.Service + users *countingUsers + alice, bob uuid.UUID + eve uuid.UUID + ctx context.Context +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + ctx := context.Background() + users := &countingUsers{Service: user.NewService(userstore.NewUsers(db.Pool))} + register := func(name string) uuid.UUID { + u, err := users.Register(ctx, name, name, "correct-horse-battery") + if err != nil { + t.Fatal(err) + } + return u.ID + } + return fixture{ + svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users), + users: users, + alice: register("alice"), + bob: register("bob"), + eve: register("eve"), + ctx: ctx, + } +} + +func TestCreateChatChecksMembersInOneCall(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "Проект", []uuid.UUID{f.eve, f.alice, f.bob}) + if err != nil { + t.Fatalf("Create error: %v", err) + } + if c.Title != "Проект" || len(c.MemberIDs) != 3 { + t.Fatalf("unexpected chat %+v", c) + } + if got := f.users.calls.Load(); got != 1 { + t.Fatalf("member existence must be checked with one batch call, got %d", got) + } + got, err := f.svc.Get(f.ctx, f.bob, c.ID) + if err != nil { + t.Fatalf("member Get error: %v", err) + } + if fmt.Sprint(got.MemberIDs) != fmt.Sprint(c.MemberIDs) || !got.CreatedAt.Equal(c.CreatedAt) || got.Title != c.Title { + t.Fatalf("Get = %+v, want what Create returned %+v", got, c) + } +} + +func TestCreateChatRejectsUnknownMember(t *testing.T) { + f := newFixture(t) + _, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, ident.New()}) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("kind = %v, want KindInvalid (%v)", apperr.KindOf(err), err) + } + if !strings.Contains(err.Error(), "member_ids") { + t.Fatalf("message must name the field: %v", err) + } +} + +func TestCreateChatRejectsInvalidInputBeforeAnyLookup(t *testing.T) { + f := newFixture(t) + cases := map[string]struct { + title string + members []uuid.UUID + }{ + "blank title": {" ", []uuid.UUID{f.alice}}, + "missing creator": {"t", []uuid.UUID{f.bob}}, + "duplicate": {"t", []uuid.UUID{f.alice, f.alice}}, + } + for name, tc := range cases { + _, err := f.svc.Create(f.ctx, f.alice, tc.title, tc.members) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: kind = %v, want KindInvalid (%v)", name, apperr.KindOf(err), err) + } + } + if got := f.users.calls.Load(); got != 0 { + t.Fatalf("invalid input must be rejected before any user lookup, got %d calls", got) + } +} + +func TestGetHidesForeignAndUnknownChatsAlike(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, foreign := f.svc.Get(f.ctx, f.eve, c.ID) + _, unknown := f.svc.Get(f.ctx, f.eve, ident.New()) + if !errors.Is(foreign, chat.ErrNotFound) || !errors.Is(unknown, chat.ErrNotFound) { + t.Fatalf("foreign = %v, unknown = %v; want both ErrNotFound", foreign, unknown) + } + if apperr.KindOf(foreign) != apperr.KindNotFound { + t.Fatalf("ErrNotFound must carry KindNotFound, got %v", apperr.KindOf(foreign)) + } +} + +func TestListPagesOnlyOwnChats(t *testing.T) { + f := newFixture(t) + var mine []uuid.UUID + for range 3 { + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + mine = append(mine, c.ID) + if _, err := f.svc.Create(f.ctx, f.eve, "t", []uuid.UUID{f.eve}); err != nil { + t.Fatal(err) + } + } + page, err := f.svc.List(f.ctx, f.alice, nil, 2) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 2 || page.Next == nil || *page.Next != mine[1] { + t.Fatalf("first page = %+v", page) + } + if len(page.Items[0].MemberIDs) != 2 { + t.Fatalf("listed chats must carry their members: %+v", page.Items[0]) + } + rest, err := f.svc.List(f.ctx, f.alice, page.Next, 2) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || rest.Items[0].ID != mine[2] || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + all, err := f.svc.List(f.ctx, f.bob, nil, 100) + if err != nil || len(all.Items) != 3 || all.Next != nil { + t.Fatalf("bob page = %+v, %v", all, err) + } +} + +func TestSendChecksAccessBeforeValidation(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, err = f.svc.Send(f.ctx, f.eve, c.ID, " ") + if !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err) + } + _, err = f.svc.Send(f.ctx, f.bob, c.ID, " ") + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err)) + } + msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "Привет 👋") + if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" { + t.Fatalf("Send = %+v, %v", msg, err) + } +} + +func TestHistoryPagesAndHidesForeignChat(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "1") + second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "2") + + page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0] != first || page.Next == nil || *page.Next != first.ID { + t.Fatalf("first page = %+v", page) + } + rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || rest.Items[0] != second || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider history: err = %v, want ErrNotFound", err) + } + empty, err := f.svc.History(f.ctx, f.alice, c.ID, &second.ID, 10) + if err != nil || empty.Items == nil || len(empty.Items) != 0 || empty.Next != nil { + t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go new file mode 100644 index 00000000..e3634a38 --- /dev/null +++ b/internal/chat/store/postgres.go @@ -0,0 +1,153 @@ +// Package store holds the repository implementations of the chats module. +// +// Keyset pages use `id > COALESCE($after, nil uuid)`: every UUIDv7 is +// greater than the nil UUID, so one query serves the first page and the +// following ones, and the condition stays a range the index can scan. Ids +// are UUIDv7 and created_at derives from them, so ordering by id is the +// contract order (created_at, id). Errors pass through postgres.Classify, so +// an unreachable database surfaces as unavailability. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/chat" + "messenger/internal/platform/postgres" +) + +type Chats struct { + pool *pgxpool.Pool +} + +var _ chat.ChatRepository = (*Chats)(nil) + +func NewChats(pool *pgxpool.Pool) *Chats { + return &Chats{pool: pool} +} + +func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, + `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`, + c.ID, c.Title, c.CreatedAt); err != nil { + return fmt.Errorf("insert chat: %w", err) + } + if _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id) SELECT $1, unnest($2::uuid[])`, + c.ID, memberIDs); err != nil { + return fmt.Errorf("insert chat members: %w", err) + } + return nil + })) +} + +func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (chat.ChatWithMembers, error) { + row := r.pool.QueryRow(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chats c + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID) + c, err := scanChat(row) + if errors.Is(err, pgx.ErrNoRows) { + return chat.ChatWithMembers{}, chat.ErrNotFound + } + if err != nil { + return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", postgres.Classify(err)) + } + return c, nil +} + +func (r *Chats) IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) { + var ok bool + err := r.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + if err != nil { + return false, fmt.Errorf("check membership: %w", postgres.Classify(err)) + } + return ok, nil +} + +func (r *Chats) ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]chat.ChatWithMembers, error) { + rows, err := r.pool.Query(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chat_members own + JOIN chats c ON c.id = own.chat_id + WHERE own.user_id = $1 + AND own.chat_id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY own.chat_id + LIMIT $3`, + userID, after, limit) + if err != nil { + return nil, fmt.Errorf("select chats: %w", postgres.Classify(err)) + } + chats, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.ChatWithMembers, error) { + return scanChat(row) + }) + if err != nil { + return nil, fmt.Errorf("read chats: %w", postgres.Classify(err)) + } + return chats, nil +} + +func scanChat(row pgx.Row) (chat.ChatWithMembers, error) { + var c chat.ChatWithMembers + if err := row.Scan(&c.ID, &c.Title, &c.CreatedAt, &c.MemberIDs); err != nil { + return chat.ChatWithMembers{}, err + } + return c, nil +} + +type Messages struct { + pool *pgxpool.Pool +} + +var _ chat.MessageRepository = (*Messages)(nil) + +func NewMessages(pool *pgxpool.Pool) *Messages { + return &Messages{pool: pool} +} + +func (r *Messages) Append(ctx context.Context, m chat.Message) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`, + m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt) + if err != nil { + return fmt.Errorf("insert message: %w", postgres.Classify(err)) + } + return nil +} + +// ListByChat is the history query; scripts/explain-history.sql runs the same +// statement for the EXPLAIN evidence. +func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) { + rows, err := r.pool.Query(ctx, ` + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY id + LIMIT $3`, + chatID, after, limit) + if err != nil { + return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) + } + messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { + var m chat.Message + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + return m, err + }) + if err != nil { + return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) + } + return messages, nil +} diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go new file mode 100644 index 00000000..5b005ea7 --- /dev/null +++ b/internal/chat/store/postgres_test.go @@ -0,0 +1,313 @@ +package store_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "sync" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +type fixture struct { + ctx context.Context + chats *store.Chats + messages *store.Messages +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + return fixture{ctx: context.Background(), chats: store.NewChats(db.Pool), messages: store.NewMessages(db.Pool)} +} + +// mustUser stores a user row: chats and messages reference users by FK. +func (f fixture) mustUser(t *testing.T) uuid.UUID { + t.Helper() + id := ident.New() + u, err := user.New(id, "u"+id.String()[24:], "User") + if err != nil { + t.Fatal(err) + } + u.PasswordHash = "x" + if err := userstore.NewUsers(db.Pool).Create(f.ctx, u); err != nil { + t.Fatal(err) + } + return id +} + +func (f fixture) mustChat(t *testing.T, members ...uuid.UUID) chat.Chat { + t.Helper() + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, members); err != nil { + t.Fatal(err) + } + return c +} + +func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), chatID, sender, "text") + if err != nil { + t.Fatal(err) + } + return m +} + +func TestChatCreateThenGetForMember(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + + got, err := f.chats.GetForMember(f.ctx, c.ID, bob) + if err != nil { + t.Fatal(err) + } + if got.Chat != c || len(got.MemberIDs) != 2 || got.MemberIDs[0] != alice || got.MemberIDs[1] != bob { + t.Fatalf("GetForMember = %+v, want %+v with members [alice bob]", got, c) + } + if _, err := f.chats.GetForMember(f.ctx, c.ID, f.mustUser(t)); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider must get ErrNotFound, got %v", err) + } + for who, want := range map[uuid.UUID]bool{alice: true, bob: true, ident.New(): false} { + ok, err := f.chats.IsMember(f.ctx, c.ID, who) + if err != nil || ok != want { + t.Fatalf("IsMember(%s) = %v, %v; want %v", who, ok, err, want) + } + } +} + +// The chat row and its members are written in one transaction: a failing +// member insert must leave no chat behind. +func TestCreateRollsBackWhenMemberInsertFails(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c, err := chat.NewChat(ident.New(), "half-made") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, ident.New()}); err == nil { + t.Fatal("Create with an unknown member must fail") + } + var chats, members int + if err := db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members)`).Scan(&chats, &members); err != nil { + t.Fatal(err) + } + if chats != 0 || members != 0 { + t.Fatalf("chats = %d, members = %d after a failed create; want 0 and 0", chats, members) + } +} + +func TestChatUnknownID(t *testing.T) { + f := newFixture(t) + if _, err := f.chats.GetForMember(f.ctx, ident.New(), ident.New()); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("GetForMember err = %v, want ErrNotFound", err) + } + if ok, err := f.chats.IsMember(f.ctx, ident.New(), ident.New()); err != nil || ok { + t.Fatalf("IsMember on unknown chat = %v, %v; want false, nil", ok, err) + } +} + +func TestListByMemberFiltersOrdersAndPages(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + var mine []chat.Chat + for range 5 { + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + mine = append(mine, c) + // A chat without alice must never show up in her listing. + f.mustChat(t, bob) + } + // Insert the oldest chat last to prove the store sorts, not appends. + for _, c := range append(mine[1:], mine[0]) { + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, bob}); err != nil { + t.Fatal(err) + } + } + + page, err := f.chats.ListByMember(f.ctx, alice, nil, 3) + if err != nil { + t.Fatal(err) + } + if len(page) != 3 || page[0].ID != mine[0].ID || page[2].ID != mine[2].ID { + t.Fatalf("first page mismatch: %v", page) + } + if len(page[0].MemberIDs) != 2 || page[0].MemberIDs[0] != alice { + t.Fatalf("listing must carry members in id order: %+v", page[0]) + } + last := page[2].ID + rest, err := f.chats.ListByMember(f.ctx, alice, &last, 10) + if err != nil { + t.Fatal(err) + } + if len(rest) != 2 || rest[0].ID != mine[3].ID || rest[1].ID != mine[4].ID { + t.Fatalf("second page mismatch: %v", rest) + } + none, err := f.chats.ListByMember(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if none == nil || len(none) != 0 { + t.Fatalf("unknown member must get an empty non-nil slice, got %#v", none) + } +} + +func TestListByChatUnknownChatIsEmpty(t *testing.T) { + f := newFixture(t) + got, err := f.messages.ListByChat(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if got == nil || len(got) != 0 { + t.Fatalf("want empty non-nil slice, got %#v", got) + } +} + +func TestAppendOutOfOrderIsListedByID(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{third, first, second} { + if err := f.messages.Append(f.ctx, m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0] != first || got[1] != second || got[2] != third { + t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got) + } +} + +func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const total = 250 + var expected []uuid.UUID + for range total { + m := f.newMessage(t, c.ID, alice) + expected = append(expected, m.ID) + if err := f.messages.Append(f.ctx, m); err != nil { + t.Fatal(err) + } + } + // Another chat's history must not leak into this one. + other := f.mustChat(t, alice) + if err := f.messages.Append(f.ctx, f.newMessage(t, other.ID, alice)); err != nil { + t.Fatal(err) + } + for _, limit := range []int{1, 7, 100} { + var seen []uuid.UUID + var after *uuid.UUID + for { + page, err := f.messages.ListByChat(f.ctx, c.ID, after, limit) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + if len(page) > limit { + t.Fatalf("limit %d: page has %d items", limit, len(page)) + } + for _, m := range page { + seen = append(seen, m.ID) + } + last := page[len(page)-1].ID + after = &last + } + if len(seen) != total { + t.Fatalf("limit %d: walked %d messages, want %d", limit, len(seen), total) + } + for i := range seen { + if seen[i] != expected[i] { + t.Fatalf("limit %d: position %d has %s, want %s", limit, i, seen[i], expected[i]) + } + } + } +} + +func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.newMessage(t, c.ID, alice) + gap := ident.New() // never stored, but sorts between a and b + b := f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{a, b} { + if err := f.messages.Append(f.ctx, m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, &gap, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != b.ID { + t.Fatalf("want only b after the gap id, got %v", got) + } +} + +func TestConcurrentAppendsKeepEveryMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const clients, perClient = 10, 10 + var wg sync.WaitGroup + for range clients { + wg.Go(func() { + for range perClient { + if err := f.messages.Append(f.ctx, f.newMessage(t, c.ID, alice)); err != nil { + t.Error(err) + } + } + }) + } + wg.Wait() + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, clients*perClient+1) + if err != nil { + t.Fatal(err) + } + if len(got) != clients*perClient { + t.Fatalf("stored %d messages, want %d", len(got), clients*perClient) + } + for i := 1; i < len(got); i++ { + if bytes.Compare(got[i-1].ID[:], got[i].ID[:]) >= 0 { + t.Fatalf("history out of order at %d", i) + } + } +} diff --git a/internal/health/handler.go b/internal/health/handler.go new file mode 100644 index 00000000..9b14e880 --- /dev/null +++ b/internal/health/handler.go @@ -0,0 +1,77 @@ +// Package health serves the probes. Liveness only proves the process +// answers. Readiness means the process can serve the synchronous API: it is +// switched off during shutdown, so an orchestrator stops routing new traffic +// before the listener closes, and it fails while a critical dependency is +// unreachable. +package health + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +// checkTimeout is shared by all checks of one probe, so the probe answers +// within it even when every dependency hangs; the Docker healthcheck allows +// two seconds. +const checkTimeout = time.Second + +// Check is a dependency the API cannot serve requests without. +type Check struct { + Name string + Probe func(ctx context.Context) error +} + +type Handler struct { + ready atomic.Bool + checks []Check +} + +type status struct { + Status string `json:"status"` +} + +func New(checks ...Check) *Handler { + h := &Handler{checks: checks} + h.ready.Store(true) + return h +} + +func (h *Handler) SetReady(ready bool) { + h.ready.Store(ready) +} + +func (h *Handler) IsReady() bool { + return h.ready.Load() +} + +func (h *Handler) Register(r gin.IRouter) { + r.GET("/health/live", h.live) + r.GET("/health/ready", h.readyz) +} + +func (h *Handler) live(c *gin.Context) { + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} + +func (h *Handler) readyz(c *gin.Context) { + if !h.ready.Load() { + httpx.Error(c, apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(c.Request.Context(), checkTimeout) + defer cancel() + for _, check := range h.checks { + if err := check.Probe(ctx); err != nil { + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, check.Name+" is unavailable", err)) + return + } + } + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} diff --git a/internal/platform/apperr/apperr.go b/internal/platform/apperr/apperr.go new file mode 100644 index 00000000..8de0ff3e --- /dev/null +++ b/internal/platform/apperr/apperr.go @@ -0,0 +1,54 @@ +// Package apperr is the application's error vocabulary: every module reports +// failures as a Kind plus a message that is safe to show to clients, and the +// transport layer owns the only mapping from Kind to HTTP status. Causes are +// kept for logs but never rendered. +package apperr + +import "errors" + +type Kind int + +const ( + KindInternal Kind = iota + KindInvalid + KindUnauthorized + KindNotFound + KindConflict + KindPayloadTooLarge + KindUnsupportedMedia + KindUnavailable +) + +type Error struct { + Kind Kind + Message string + Cause error +} + +func (e *Error) Error() string { return e.Message } + +func (e *Error) Unwrap() error { return e.Cause } + +func New(kind Kind, message string) *Error { + return &Error{Kind: kind, Message: message} +} + +func Wrap(kind Kind, message string, cause error) *Error { + return &Error{Kind: kind, Message: message, Cause: cause} +} + +// KindOf classifies any error; foreign errors are internal. +func KindOf(err error) Kind { + if e, ok := errors.AsType[*Error](err); ok { + return e.Kind + } + return KindInternal +} + +func Invalid(message string) *Error { return New(KindInvalid, message) } +func Unauthorized(message string) *Error { return New(KindUnauthorized, message) } +func NotFound(message string) *Error { return New(KindNotFound, message) } +func Conflict(message string) *Error { return New(KindConflict, message) } +func PayloadTooLarge(message string) *Error { return New(KindPayloadTooLarge, message) } +func UnsupportedMedia(message string) *Error { return New(KindUnsupportedMedia, message) } +func Unavailable(message string) *Error { return New(KindUnavailable, message) } diff --git a/internal/platform/apperr/apperr_test.go b/internal/platform/apperr/apperr_test.go new file mode 100644 index 00000000..6de347f7 --- /dev/null +++ b/internal/platform/apperr/apperr_test.go @@ -0,0 +1,49 @@ +package apperr_test + +import ( + "errors" + "fmt" + "testing" + + "messenger/internal/platform/apperr" +) + +func TestKindOfReturnsKindForApplicationError(t *testing.T) { + err := apperr.NotFound("chat not found") + if got := apperr.KindOf(err); got != apperr.KindNotFound { + t.Fatalf("KindOf = %v, want KindNotFound", got) + } + if err.Error() != "chat not found" { + t.Fatalf("Error() = %q", err.Error()) + } +} + +func TestKindOfSeesThroughWrapping(t *testing.T) { + err := fmt.Errorf("service: %w", apperr.Conflict("username taken")) + if got := apperr.KindOf(err); got != apperr.KindConflict { + t.Fatalf("KindOf = %v, want KindConflict", got) + } +} + +func TestKindOfDefaultsToInternal(t *testing.T) { + if got := apperr.KindOf(errors.New("boom")); got != apperr.KindInternal { + t.Fatalf("KindOf = %v, want KindInternal", got) + } + if got := apperr.KindOf(nil); got != apperr.KindInternal { + t.Fatalf("KindOf(nil) = %v, want KindInternal", got) + } +} + +func TestWrapKeepsCause(t *testing.T) { + cause := errors.New("cause") + err := apperr.Wrap(apperr.KindInvalid, "bad input", cause) + if !errors.Is(err, cause) { + t.Fatal("wrapped error must unwrap to cause") + } + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatal("wrapped error must keep its kind") + } + if err.Error() != "bad input" { + t.Fatalf("Error() = %q, want public message only", err.Error()) + } +} diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go new file mode 100644 index 00000000..30100b71 --- /dev/null +++ b/internal/platform/config/config.go @@ -0,0 +1,118 @@ +// Package config reads the process configuration from the environment. +// Settings have defaults suited to the lab stand, except credentials, which +// are required and read from files named by *_FILE variables so that their +// values never appear in the environment. An invalid value stops the process +// at start-up with a message naming the variable, never the secret. +package config + +import ( + "fmt" + "log/slog" + "net" + "os" + "strconv" + "strings" + "time" +) + +type Config struct { + Host string + Port int + Env string + LogLevel slog.Level + ShutdownTimeout time.Duration + MaxBodyBytes int64 + DatabaseURL string + SessionStoreURL string + SessionTTL time.Duration +} + +// Lookup reads one variable, reporting whether it was set; os.LookupEnv +// in production, a map in tests. +type Lookup func(key string) (string, bool) + +func Load(lookup Lookup) (Config, error) { + cfg := Config{ + Host: "0.0.0.0", + Port: 8080, + Env: "lab", + LogLevel: slog.LevelInfo, + ShutdownTimeout: 8 * time.Second, + MaxBodyBytes: 64 * 1024, + SessionTTL: time.Hour, + } + if v, ok := lookup("APP_HOST"); ok { + if strings.TrimSpace(v) == "" { + return Config{}, fmt.Errorf("APP_HOST must not be empty") + } + cfg.Host = v + } + if v, ok := lookup("APP_PORT"); ok { + port, err := strconv.Atoi(v) + if err != nil || port < 1 || port > 65535 { + return Config{}, fmt.Errorf("APP_PORT must be an integer between 1 and 65535, got %q", v) + } + cfg.Port = port + } + if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { + cfg.Env = v + } + if v, ok := lookup("APP_LOG_LEVEL"); ok { + var level slog.Level + if err := level.UnmarshalText([]byte(v)); err != nil { + return Config{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + } + cfg.LogLevel = level + } + if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return Config{}, fmt.Errorf("APP_SHUTDOWN_TIMEOUT must be a positive duration such as 8s, got %q", v) + } + cfg.ShutdownTimeout = d + } + if v, ok := lookup("APP_MAX_BODY_BYTES"); ok { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n <= 0 { + return Config{}, fmt.Errorf("APP_MAX_BODY_BYTES must be a positive integer, got %q", v) + } + cfg.MaxBodyBytes = n + } + if v, ok := lookup("SESSION_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.SessionTTL = time.Duration(n) * time.Second + } + var err error + if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + return Config{}, err + } + if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { + return Config{}, err + } + return cfg, nil +} + +// secret reads the file named by the variable; surrounding whitespace, +// including the trailing newline editors add, is dropped. +func secret(lookup Lookup, key string) (string, error) { + path, ok := lookup(key) + if !ok || strings.TrimSpace(path) == "" { + return "", fmt.Errorf("%s must name a file with the value", key) + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("%s: cannot read %s: %w", key, path, err) + } + value := strings.TrimSpace(string(raw)) + if value == "" { + return "", fmt.Errorf("%s: file %s is empty", key, path) + } + return value, nil +} + +func (c Config) Addr() string { + return net.JoinHostPort(c.Host, strconv.Itoa(c.Port)) +} diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go new file mode 100644 index 00000000..2288dcd2 --- /dev/null +++ b/internal/platform/config/config_test.go @@ -0,0 +1,135 @@ +package config_test + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/platform/config" +) + +func env(values map[string]string) func(string) (string, bool) { + return func(key string) (string, bool) { + v, ok := values[key] + return v, ok + } +} + +func secretFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +const ( + databaseURL = "postgres://app:s3cret@db:5432/messenger" + sessionStoreURL = "redis://:s3cret@valkey:6379/0" +) + +// required holds the variables without defaults, merged with extra. +func required(t *testing.T, extra map[string]string) map[string]string { + t.Helper() + values := map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "SESSION_STORE_URL_FILE": secretFile(t, sessionStoreURL+"\r\n"), + } + for k, v := range extra { + values[k] = v + } + return values +} + +func TestDefaults(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "0.0.0.0:8080" { + t.Fatalf("Addr = %q", cfg.Addr()) + } + if cfg.Env != "lab" || cfg.LogLevel != slog.LevelInfo { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL { + t.Fatalf("urls = %q, %q; want the file contents without line endings", cfg.DatabaseURL, cfg.SessionStoreURL) + } + if cfg.SessionTTL != time.Hour { + t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL) + } +} + +func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { + for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} { + cases := map[string]func(map[string]string){ + "unset": func(v map[string]string) { delete(v, key) }, + "empty path": func(v map[string]string) { v[key] = "" }, + "no file": func(v map[string]string) { v[key] = filepath.Join(t.TempDir(), "missing") }, + "blank file": func(v map[string]string) { v[key] = secretFile(t, " \r\n") }, + } + for name, breakIt := range cases { + values := required(t, nil) + breakIt(values) + _, err := config.Load(env(values)) + if err == nil { + t.Errorf("%s %s: want error", key, name) + continue + } + if !strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "s3cret") { + t.Errorf("%s %s: error must name the variable and hide the secret: %v", key, name, err) + } + } + } +} + +func TestOverrides(t *testing.T) { + cfg, err := config.Load(env(required(t, map[string]string{ + "APP_HOST": "127.0.0.1", + "APP_PORT": "9090", + "APP_ENV": "dev", + "APP_LOG_LEVEL": "debug", + "APP_SHUTDOWN_TIMEOUT": "3s", + "APP_MAX_BODY_BYTES": "1024", + "SESSION_TTL_SECONDS": "300", + }))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.LogLevel != slog.LevelDebug { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { + t.Fatalf("cfg = %+v", cfg) + } +} + +func TestInvalidValuesAreReported(t *testing.T) { + cases := map[string]map[string]string{ + "port text": {"APP_PORT": "abc"}, + "port zero": {"APP_PORT": "0"}, + "port too big": {"APP_PORT": "70000"}, + "level": {"APP_LOG_LEVEL": "loud"}, + "timeout": {"APP_SHUTDOWN_TIMEOUT": "soon"}, + "timeout negative": {"APP_SHUTDOWN_TIMEOUT": "-1s"}, + "body zero": {"APP_MAX_BODY_BYTES": "0"}, + "body text": {"APP_MAX_BODY_BYTES": "big"}, + "empty host": {"APP_HOST": ""}, + "ttl zero": {"SESSION_TTL_SECONDS": "0"}, + "ttl negative": {"SESSION_TTL_SECONDS": "-5"}, + "ttl text": {"SESSION_TTL_SECONDS": "hour"}, + "ttl fraction": {"SESSION_TTL_SECONDS": "1.5"}, + } + for name, values := range cases { + if _, err := config.Load(env(required(t, values))); err == nil { + t.Errorf("%s: want error", name) + } + } +} diff --git a/internal/platform/httpx/decode.go b/internal/platform/httpx/decode.go new file mode 100644 index 00000000..ce8eeae7 --- /dev/null +++ b/internal/platform/httpx/decode.go @@ -0,0 +1,70 @@ +package httpx + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strings" + "unicode/utf8" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +// DecodeJSON reads exactly one JSON object into dst and enforces what gin's +// binding does not: the media type must be application/json (415), invalid +// UTF-8, unknown fields, wrong types, malformed syntax and trailing data are +// 400, an oversized body is 413, and messages never expose Go type names. +func DecodeJSON(c *gin.Context, dst any) error { + mediaType, _, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || mediaType != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + // The body is already capped by the bodyLimit middleware, so reading it + // whole is bounded. + body, err := io.ReadAll(c.Request.Body) + if err != nil { + return decodeError(err) + } + // encoding/json silently replaces invalid UTF-8 with U+FFFD; the contract + // and RFC 8259 require UTF-8, so such a body is rejected, not repaired. + if !utf8.Valid(body) { + return apperr.Invalid("request body must be valid UTF-8") + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return decodeError(err) + } + if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return apperr.Invalid("unexpected data after JSON body") + } + return nil +} + +func decodeError(err error) error { + if tooLarge, ok := errors.AsType[*http.MaxBytesError](err); ok { + return apperr.Wrap(apperr.KindPayloadTooLarge, fmt.Sprintf("request body must be at most %d bytes", tooLarge.Limit), err) + } + if errors.Is(err, io.EOF) { + return apperr.Wrap(apperr.KindInvalid, "request body is required", err) + } + if wrongType, ok := errors.AsType[*json.UnmarshalTypeError](err); ok { + if wrongType.Field == "" { + return apperr.Wrap(apperr.KindInvalid, "request body must be a JSON object", err) + } + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("field %q has wrong type", wrongType.Field), err) + } + // encoding/json reports unknown fields as a plain error without a type. + const unknownPrefix = "json: unknown field " + if msg := err.Error(); strings.HasPrefix(msg, unknownPrefix) { + field := strings.Trim(strings.TrimPrefix(msg, unknownPrefix), `"`) + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("unknown field %q", field), err) + } + return apperr.Wrap(apperr.KindInvalid, "malformed JSON body", err) +} diff --git a/internal/platform/httpx/engine.go b/internal/platform/httpx/engine.go new file mode 100644 index 00000000..ba56a897 --- /dev/null +++ b/internal/platform/httpx/engine.go @@ -0,0 +1,49 @@ +// Package httpx is the transport layer over gin: request ids, access +// logging, panic recovery, body limits, strict JSON decoding and the single +// error format of the API contract. gin is used for routing and the +// middleware chain; everything here exists because the contract demands a +// specific behaviour that gin's own helpers do not provide. +package httpx + +import ( + "log/slog" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +const ( + keyRequestID = "messenger.request_id" + keyLogger = "messenger.logger" +) + +// NewEngine builds an engine whose every response, including routing +// failures and panics, carries X-Request-Id and the contract error body. +func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine { + gin.SetMode(gin.ReleaseMode) + e := gin.New() + // Redirects would answer with text/plain and without a request id. + e.RedirectTrailingSlash = false + e.RedirectFixedPath = false + e.HandleMethodNotAllowed = true + + e.Use(requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) + e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) }) + e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") }) + return e +} + +func RequestID(c *gin.Context) string { + return c.GetString(keyRequestID) +} + +func logger(c *gin.Context) *slog.Logger { + if l, ok := c.Get(keyLogger); ok { + if log, ok := l.(*slog.Logger); ok { + return log + } + } + return slog.Default() +} diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go new file mode 100644 index 00000000..a745d5a9 --- /dev/null +++ b/internal/platform/httpx/httpx_test.go @@ -0,0 +1,376 @@ +package httpx_test + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type payload struct { + Text string `json:"text"` + Count int `json:"count"` +} + +func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine { + t.Helper() + return newEngineAt(t, logs, slog.LevelDebug) +} + +func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine { + t.Helper() + if logs == nil { + logs = &bytes.Buffer{} + } + log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level})) + e := httpx.NewEngine(log, 64) + e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": ""}) }) + e.GET("/panic", func(c *gin.Context) { panic("secret detail") }) + e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) }) + e.GET("/notfound", func(c *gin.Context) { httpx.Error(c, apperr.NotFound("chat not found")) }) + e.GET("/who", func(c *gin.Context) { + httpx.SetUserID(c, uuid.MustParse("11111111-1111-4111-8111-111111111111")) + id, ok := httpx.UserID(c) + httpx.JSON(c, http.StatusOK, gin.H{"id": id.String(), "ok": ok}) + }) + e.POST("/decode", func(c *gin.Context) { + var p payload + if err := httpx.DecodeJSON(c, &p); err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, p) + }) + return e +} + +func do(t *testing.T, e *gin.Engine, method, path string, body string, headers map[string]string) (*httptest.ResponseRecorder, errorBody) { + t.Helper() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, path, reader) + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + var eb errorBody + if rec.Code >= 400 { + if err := json.Unmarshal(rec.Body.Bytes(), &eb); err != nil { + t.Fatalf("%s %s: error body is not JSON: %v: %s", method, path, err, rec.Body.String()) + } + } + return rec, eb +} + +func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, status int, code string) { + t.Helper() + if rec.Code != status { + t.Fatalf("status = %d, want %d (body %s)", rec.Code, status, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Fatalf("Content-Type = %q", ct) + } + rid := rec.Header().Get("X-Request-Id") + if rid == "" { + t.Fatal("missing X-Request-Id header") + } + if eb.Error.RequestID != rid { + t.Fatalf("body request_id %q != header %q", eb.Error.RequestID, rid) + } + if eb.Error.Code != code || eb.Error.Message == "" { + t.Fatalf("error = %+v, want code %q with a message", eb.Error, code) + } +} + +func TestUnknownRouteIsJSON404(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestTrailingSlashIsNotRedirected(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/ok/", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestWrongMethodIsJSON405WithAllow(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodDelete, "/ok", "", nil) + assertError(t, rec, eb, http.StatusMethodNotAllowed, "invalid_request") + if !strings.Contains(rec.Header().Get("Allow"), "GET") { + t.Fatalf("Allow = %q, want GET", rec.Header().Get("Allow")) + } +} + +func TestPanicBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/panic", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "secret detail") { + t.Fatal("panic value leaked to the client") + } + if !strings.Contains(logs.String(), "secret detail") { + t.Fatal("panic must be logged for operators") + } +} + +func TestForeignErrorBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/fail", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "password") { + t.Fatal("internal error text leaked to the client") + } + if !strings.Contains(logs.String(), "db password leaked") { + t.Fatal("cause must be logged for operators") + } +} + +func TestApplicationErrorMapping(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/notfound", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") + if eb.Error.Message != "chat not found" { + t.Fatalf("message = %q", eb.Error.Message) + } +} + +func TestSuccessResponseHeadersAndEscaping(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json; charset=utf-8" { + t.Fatalf("Content-Type = %q", ct) + } + if rec.Header().Get("X-Request-Id") == "" { + t.Fatal("missing X-Request-Id on success") + } + if !strings.Contains(rec.Body.String(), `""`) { + t.Fatalf("HTML escaping must be off, body %s", rec.Body.String()) + } +} + +func TestRequestIDIsGeneratedNotEchoed(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": "client-chosen"}) + if got := rec.Header().Get("X-Request-Id"); got == "" || got == "client-chosen" { + t.Fatalf("X-Request-Id = %q, want a server generated value", got) + } +} + +func TestDecodeJSONHappyPath(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodPost, "/decode", `{"text":"Привет","count":2}`, map[string]string{"Content-Type": "application/json; charset=utf-8"}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"Привет"`) { + t.Fatalf("status %d body %s", rec.Code, rec.Body.String()) + } +} + +func TestDecodeJSONFailures(t *testing.T) { + jsonCT := map[string]string{"Content-Type": "application/json"} + cases := []struct { + name string + body string + headers map[string]string + status int + code string + contain string + }{ + {"no content type", `{"text":"x"}`, nil, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"wrong content type", `{"text":"x"}`, map[string]string{"Content-Type": "text/plain"}, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"empty body", "", jsonCT, http.StatusBadRequest, "invalid_request", "body"}, + {"malformed", `{"text":`, jsonCT, http.StatusBadRequest, "invalid_request", "malformed"}, + {"invalid utf-8", "{\"text\":\"bad \xff byte\"}", jsonCT, http.StatusBadRequest, "invalid_request", "UTF-8"}, + {"unknown field", `{"text":"x","extra":1}`, jsonCT, http.StatusBadRequest, "invalid_request", "extra"}, + {"wrong type", `{"text":123}`, jsonCT, http.StatusBadRequest, "invalid_request", "text"}, + {"trailing data", `{"text":"x"} {"text":"y"}`, jsonCT, http.StatusBadRequest, "invalid_request", "after"}, + {"array instead of object", `[1,2]`, jsonCT, http.StatusBadRequest, "invalid_request", ""}, + {"too large", `{"text":"` + strings.Repeat("x", 100) + `"}`, jsonCT, http.StatusRequestEntityTooLarge, "payload_too_large", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodPost, "/decode", tc.body, tc.headers) + assertError(t, rec, eb, tc.status, tc.code) + if tc.contain != "" && !strings.Contains(eb.Error.Message, tc.contain) { + t.Fatalf("message %q must mention %q", eb.Error.Message, tc.contain) + } + if strings.Contains(eb.Error.Message, "httpx_test") || strings.Contains(eb.Error.Message, "payload") { + t.Fatalf("message must not leak Go type names: %q", eb.Error.Message) + } + }) + } +} + +func TestAccessLogHasRequestFieldsButNoBody(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + rec, _ := do(t, e, http.MethodPost, "/decode", `{"text":"confidential"}`, map[string]string{"Content-Type": "application/json"}) + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + for _, key := range []string{"request_id", "method", "path", "status", "duration_ms"} { + if _, ok := entry[key]; !ok { + t.Errorf("log entry missing %q: %s", key, line) + } + } + if entry["request_id"] != rec.Header().Get("X-Request-Id") { + t.Errorf("log request_id %v != header %q", entry["request_id"], rec.Header().Get("X-Request-Id")) + } + if strings.Contains(logs.String(), "confidential") { + t.Fatal("request body must never be logged") + } +} + +func TestAccessLogLevelFollowsStatus(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngineAt(t, logs, slog.LevelInfo) + + do(t, e, http.MethodGet, "/ok", "", nil) + if strings.Contains(logs.String(), `"msg":"request"`) { + t.Fatalf("successful requests must be logged at debug level only, got %s", logs.String()) + } + do(t, e, http.MethodGet, "/notfound", "", nil) + if entry := logEntry(t, logs); entry["level"] != "INFO" || entry["status"] != float64(404) { + t.Fatalf("client error must be logged at info level with its status, got %v", entry) + } + do(t, e, http.MethodGet, "/fail", "", nil) + if entry := logEntry(t, logs); entry["level"] != "ERROR" || entry["status"] != float64(500) { + t.Fatalf("server error must be logged at error level with its status, got %v", entry) + } + + verbose := &bytes.Buffer{} + do(t, newEngineAt(t, verbose, slog.LevelDebug), http.MethodGet, "/ok", "", nil) + if entry := logEntry(t, verbose); entry["level"] != "DEBUG" || entry["status"] != float64(200) { + t.Fatalf("at debug level successful requests must be logged, got %v", entry) + } +} + +func TestUserIDRoundTripAndAccessLogAttribution(t *testing.T) { + logs := &bytes.Buffer{} + rec, _ := do(t, newEngine(t, logs), http.MethodGet, "/who", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"ok":true`) { + t.Fatalf("UserID must return what SetUserID stored: %d %s", rec.Code, rec.Body.String()) + } + if entry := logEntry(t, logs); entry["user_id"] != "11111111-1111-4111-8111-111111111111" { + t.Fatalf("access log must attribute the request to the user, got %v", entry) + } + anonymous := &bytes.Buffer{} + do(t, newEngine(t, anonymous), http.MethodGet, "/ok", "", nil) + if _, present := logEntry(t, anonymous)["user_id"]; present { + t.Fatal("anonymous requests must not carry user_id") + } +} + +func TestWithUserPassesTheIdentifiedCaller(t *testing.T) { + actor := uuid.MustParse("22222222-2222-4222-8222-222222222222") + e := newEngine(t, nil) + e.GET("/actor", + func(c *gin.Context) { httpx.SetUserID(c, actor) }, + httpx.WithUser(func(c *gin.Context, got uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{"id": got.String()}) + })) + rec, _ := do(t, e, http.MethodGet, "/actor", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), actor.String()) { + t.Fatalf("handler did not receive the caller: %d %s", rec.Code, rec.Body.String()) + } +} + +// Reaching WithUser without an authentication middleware in front is a +// wiring mistake, so it must not be blamed on the client with a 401. +func TestWithUserWithoutIdentityIsAWiringError(t *testing.T) { + e := newEngine(t, nil) + e.GET("/actor", httpx.WithUser(func(c *gin.Context, _ uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{}) + })) + rec, eb := do(t, e, http.MethodGet, "/actor", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") +} + +// A 5xx tells the client only the safe message; the operator must still see +// why the dependency failed, so the cause goes to the log. +func TestUnavailableCauseIsLoggedButNotReturned(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + e.GET("/down", func(c *gin.Context) { + cause := errors.New("dial tcp 10.0.0.7:5432: connect: connection refused") + httpx.Error(c, fmt.Errorf("select user: %w", apperr.Wrap(apperr.KindUnavailable, "database unavailable", cause))) + }) + rec, eb := do(t, e, http.MethodGet, "/down", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + // Neither the cause nor the wrapping context of internal operations + // ("select user: ...") may reach the client. + if eb.Error.Message != "database unavailable" { + t.Fatalf("client message = %q, want only the application message", eb.Error.Message) + } + if !strings.Contains(logs.String(), "connection refused") || !strings.Contains(logs.String(), eb.Error.RequestID) { + t.Fatalf("log must carry the cause and the request id, got:\n%s", logs.String()) + } +} + +// Everything below Timeout sees one deadline, so waiting on a dependency +// ends with the request instead of holding the client. +func TestTimeoutBoundsTheRequestContext(t *testing.T) { + e := newEngine(t, nil) + e.GET("/slow", httpx.Timeout(50*time.Millisecond), func(c *gin.Context) { + select { + case <-c.Request.Context().Done(): + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, "gave up", c.Request.Context().Err())) + case <-time.After(5 * time.Second): + httpx.JSON(c, http.StatusOK, gin.H{}) + } + }) + start := time.Now() + rec, eb := do(t, e, http.MethodGet, "/slow", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("request took %v, want it cut at the 50ms deadline", elapsed) + } +} + +func logEntry(t *testing.T, logs *bytes.Buffer) map[string]any { + t.Helper() + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + return entry +} + +func TestTimestampFormat(t *testing.T) { + ts := time.Date(2026, 9, 1, 12, 0, 0, 0, time.FixedZone("MSK", 3*3600)) + if got := httpx.Timestamp(ts); got != "2026-09-01T09:00:00.000Z" { + t.Fatalf("Timestamp = %q", got) + } + ts = time.Date(2026, 9, 1, 12, 0, 0, 7_000_000, time.UTC) + if got := httpx.Timestamp(ts); got != "2026-09-01T12:00:00.007Z" { + t.Fatalf("Timestamp = %q", got) + } +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} diff --git a/internal/platform/httpx/identity.go b/internal/platform/httpx/identity.go new file mode 100644 index 00000000..a90fd50c --- /dev/null +++ b/internal/platform/httpx/identity.go @@ -0,0 +1,42 @@ +package httpx + +import ( + "errors" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +const keyUserID = "messenger.user_id" + +// SetUserID records the authenticated caller for the handlers behind the +// authentication middleware and for the access log. +func SetUserID(c *gin.Context, id uuid.UUID) { + c.Set(keyUserID, id) +} + +func UserID(c *gin.Context) (uuid.UUID, bool) { + v, ok := c.Get(keyUserID) + if !ok { + return uuid.Nil, false + } + id, ok := v.(uuid.UUID) + return id, ok +} + +// UserHandler is a handler that needs the authenticated caller. +type UserHandler func(c *gin.Context, actor uuid.UUID) + +// WithUser adapts a UserHandler to gin. Reaching it without the +// authentication middleware in front is a wiring mistake, so it is reported +// as an internal error rather than blamed on the client with a 401. +func WithUser(h UserHandler) gin.HandlerFunc { + return func(c *gin.Context) { + actor, ok := UserID(c) + if !ok { + Error(c, errors.New("route is missing the authentication middleware")) + return + } + h(c, actor) + } +} diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go new file mode 100644 index 00000000..d153a980 --- /dev/null +++ b/internal/platform/httpx/middleware.go @@ -0,0 +1,118 @@ +package httpx + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "net/http" + "runtime/debug" + "time" + + "github.com/gin-gonic/gin" +) + +// requestID assigns a fresh identifier to every request. +// +// An incoming X-Request-Id is deliberately not reused. This service is +// client-facing: there is no trusted upstream that could have minted the +// value, so honouring it would let any caller pick an arbitrary string that +// ends up in logs (log injection, unbounded length) or collides with another +// request and corrupts log correlation. Reusing the header is the right +// choice only when the caller is a trusted proxy or a sibling service that +// propagates one id across a chain of calls. +func requestID(log *slog.Logger) gin.HandlerFunc { + return func(c *gin.Context) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + panic("httpx: random source unavailable: " + err.Error()) + } + id := hex.EncodeToString(raw[:]) + c.Set(keyRequestID, id) + c.Set(keyLogger, log) + c.Writer.Header().Set("X-Request-Id", id) + c.Next() + } +} + +// accessLog writes one line per request at a level that follows the +// outcome (2xx debug, 4xx info, 5xx error), so the ordinary log level decides +// how much traffic is recorded. Headers, query strings and bodies are never +// recorded. time.Since uses the monotonic clock, so wall clock adjustments +// cannot distort the duration. +func accessLog() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + + status := c.Writer.Status() + level := slog.LevelDebug + switch { + case status >= http.StatusInternalServerError: + level = slog.LevelError + case status >= http.StatusBadRequest: + level = slog.LevelInfo + } + log := logger(c) + ctx := c.Request.Context() + if !log.Enabled(ctx, level) { + return + } + attrs := []any{ + "request_id", RequestID(c), + "method", c.Request.Method, + "path", c.Request.URL.Path, + "status", status, + "duration_ms", float64(time.Since(start).Microseconds()) / 1000, + "bytes", c.Writer.Size(), + } + if id, ok := UserID(c); ok { + attrs = append(attrs, "user_id", id.String()) + } + log.Log(ctx, level, "request", attrs...) + } +} + +// recovery turns a panic into a contract 500; the stack goes to the log only. +func recovery() gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + if r := recover(); r != nil { + logger(c).Error("panic recovered", + "request_id", RequestID(c), + "panic", fmt.Sprint(r), + "stack", string(debug.Stack()), + ) + if !c.Writer.Written() { + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + } + c.Abort() + } + }() + c.Next() + } +} + +// Timeout bounds the whole request with one deadline: waiting for a pooled +// connection, SQL statements and session lookups all observe it, so a hung +// dependency ends in a 503 within d instead of holding the client. +func Timeout(d time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), d) + defer cancel() + c.Request = c.Request.WithContext(ctx) + c.Next() + } +} + +// bodyLimit caps request bodies; exceeding it surfaces from the JSON +// decoder as *http.MaxBytesError. +func bodyLimit(maxBytes int64) gin.HandlerFunc { + return func(c *gin.Context) { + if c.Request.Body != nil { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) + } + c.Next() + } +} diff --git a/internal/platform/httpx/respond.go b/internal/platform/httpx/respond.go new file mode 100644 index 00000000..30f49354 --- /dev/null +++ b/internal/platform/httpx/respond.go @@ -0,0 +1,87 @@ +package httpx + +import ( + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +type errorEnvelope struct { + Error errorDetail `json:"error"` +} + +type errorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type statusCode struct { + status int + code string +} + +var kinds = map[apperr.Kind]statusCode{ + apperr.KindInvalid: {http.StatusBadRequest, "invalid_request"}, + apperr.KindUnauthorized: {http.StatusUnauthorized, "unauthorized"}, + apperr.KindNotFound: {http.StatusNotFound, "not_found"}, + apperr.KindConflict: {http.StatusConflict, "conflict"}, + apperr.KindPayloadTooLarge: {http.StatusRequestEntityTooLarge, "payload_too_large"}, + apperr.KindUnsupportedMedia: {http.StatusUnsupportedMediaType, "unsupported_media_type"}, + apperr.KindUnavailable: {http.StatusServiceUnavailable, "unavailable"}, +} + +// JSON is the single place that fixes the response media type; PureJSON +// keeps non-ASCII and HTML characters verbatim. +func JSON(c *gin.Context, status int, body any) { + c.PureJSON(status, body) +} + +// Fail writes the contract error body and stops the handler chain. +func Fail(c *gin.Context, status int, code, message string) { + JSON(c, status, errorEnvelope{Error: errorDetail{ + Code: code, + Message: message, + RequestID: RequestID(c), + }}) + c.Abort() +} + +// Error maps an application error to its status. Anything that is not an +// application error is logged with its text and reported as an opaque 500, +// so internal details never reach the client. An application error shows the +// client only its own message, never the context it was wrapped in on the +// way up; when it is answered with a 5xx (a dependency outage) the full +// chain and the cause go to the log for the operator. +func Error(c *gin.Context, err error) { + appErr, isApp := errors.AsType[*apperr.Error](err) + var mapped statusCode + known := false + if isApp { + mapped, known = kinds[appErr.Kind] + } + if !known { + logger(c).Error("request failed", "request_id", RequestID(c), "error", err.Error()) + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + return + } + if mapped.status >= http.StatusInternalServerError { + attrs := []any{"request_id", RequestID(c), "error", err.Error()} + if appErr.Cause != nil { + attrs = append(attrs, "cause", appErr.Cause.Error()) + } + logger(c).Error("request failed", attrs...) + } + Fail(c, mapped.status, mapped.code, appErr.Message) +} + +// Timestamp renders RFC 3339 UTC with a fixed millisecond width: Python 3.9 +// (the oldest version the public tests support) only parses 3 or 6 fractional +// digits, and RFC3339Nano would trim trailing zeros to a variable width. +func Timestamp(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z") +} diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go new file mode 100644 index 00000000..121500d9 --- /dev/null +++ b/internal/platform/ident/ident.go @@ -0,0 +1,46 @@ +// Package ident generates and parses entity identifiers. +// +// Identifiers are UUIDv7 (RFC 9562): the first 48 bits carry a Unix +// millisecond timestamp and google/uuid keeps them strictly increasing within +// one process even when the system clock steps backwards, so lexicographic +// order of ids equals creation order. +package ident + +import ( + "errors" + "time" + + "github.com/google/uuid" +) + +var ErrInvalid = errors.New("invalid uuid") + +// New panics only if the platform random source is unavailable, which is a +// fatal environment fault rather than a request error. +func New() uuid.UUID { + id, err := uuid.NewV7() + if err != nil { + panic("ident: random source unavailable: " + err.Error()) + } + return id +} + +// TimeOf extracts the creation time carried by a UUIDv7 as UTC with +// millisecond precision. +func TimeOf(id uuid.UUID) time.Time { + sec, nsec := id.Time().UnixTime() + return time.Unix(sec, nsec).UTC() +} + +// Parse accepts only the canonical 36-character form; uuid.Parse alone would +// also take braces, urn prefixes and bare hex. +func Parse(raw string) (uuid.UUID, error) { + if len(raw) != 36 { + return uuid.Nil, ErrInvalid + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, ErrInvalid + } + return id, nil +} diff --git a/internal/platform/ident/ident_test.go b/internal/platform/ident/ident_test.go new file mode 100644 index 00000000..1ae020df --- /dev/null +++ b/internal/platform/ident/ident_test.go @@ -0,0 +1,72 @@ +package ident_test + +import ( + "bytes" + "testing" + "time" + + "messenger/internal/platform/ident" +) + +func TestNewProducesVersion7(t *testing.T) { + id := ident.New() + if got := id.Version(); got != 7 { + t.Fatalf("version = %d, want 7", got) + } +} + +func TestNewIsStrictlyIncreasingWithinProcess(t *testing.T) { + prev := ident.New() + for i := 0; i < 20000; i++ { + next := ident.New() + if bytes.Compare(next[:], prev[:]) <= 0 { + t.Fatalf("id %s is not greater than previous %s at iteration %d", next, prev, i) + } + prev = next + } +} + +func TestTimeOfMatchesWallClockAtMillisecondPrecision(t *testing.T) { + before := time.Now().Truncate(time.Millisecond) + id := ident.New() + after := time.Now().Add(time.Millisecond) + + got := ident.TimeOf(id) + if got.Location() != time.UTC { + t.Fatalf("TimeOf must return UTC, got %v", got.Location()) + } + if got.Before(before) || got.After(after) { + t.Fatalf("TimeOf = %v, want within [%v, %v]", got, before, after) + } + if got.Nanosecond()%int(time.Millisecond) != 0 { + t.Fatalf("TimeOf must be truncated to milliseconds, got %v", got) + } +} + +func TestParseAcceptsCanonicalForm(t *testing.T) { + id := ident.New() + parsed, err := ident.Parse(id.String()) + if err != nil { + t.Fatalf("Parse(%q) error: %v", id, err) + } + if parsed != id { + t.Fatalf("Parse round trip mismatch: %s != %s", parsed, id) + } +} + +func TestParseRejectsNonCanonicalForms(t *testing.T) { + id := ident.New().String() + for _, raw := range []string{ + "", + "not-a-uuid", + "{" + id + "}", + "urn:uuid:" + id, + id[:32] + id[33:], // hex without one dash + id + "0", + " " + id, + } { + if _, err := ident.Parse(raw); err == nil { + t.Errorf("Parse(%q) must fail", raw) + } + } +} diff --git a/internal/platform/pagination/cursor.go b/internal/platform/pagination/cursor.go new file mode 100644 index 00000000..65a09854 --- /dev/null +++ b/internal/platform/pagination/cursor.go @@ -0,0 +1,63 @@ +// Package pagination implements keyset paging. A cursor is opaque to the +// client and bound to the resource it pages (the chat for history, the user +// for the chat list), so a cursor from one resource is rejected on another. +// Because ids are UUIDv7 and creation time derives from the id, the keyset +// (created_at, id) collapses to the id of the last returned item. +package pagination + +import ( + "encoding/base64" + "errors" + "strconv" + + "github.com/google/uuid" +) + +const ( + DefaultLimit = 50 + MaxLimit = 100 +) + +var ( + ErrInvalidCursor = errors.New("invalid cursor") + ErrInvalidLimit = errors.New("limit must be an integer between 1 and 100") +) + +const cursorVersion = 1 + +func EncodeCursor(scope, last uuid.UUID) string { + raw := make([]byte, 0, 1+16+16) + raw = append(raw, cursorVersion) + raw = append(raw, scope[:]...) + raw = append(raw, last[:]...) + return base64.RawURLEncoding.EncodeToString(raw) +} + +func DecodeCursor(token string, scope uuid.UUID) (uuid.UUID, error) { + raw, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(raw) != 1+16+16 || raw[0] != cursorVersion { + return uuid.Nil, ErrInvalidCursor + } + if got, err := uuid.FromBytes(raw[1:17]); err != nil || got != scope { + return uuid.Nil, ErrInvalidCursor + } + last, err := uuid.FromBytes(raw[17:]) + if err != nil { + return uuid.Nil, ErrInvalidCursor + } + return last, nil +} + +// ParseLimit accepts an unsigned decimal between 1 and MaxLimit; an empty +// value means DefaultLimit. ParseUint rejects signs, whitespace and +// fractions and reports overflow, so no length guard is needed. +func ParseLimit(raw string) (int, error) { + if raw == "" { + return DefaultLimit, nil + } + n, err := strconv.ParseUint(raw, 10, 32) + if err != nil || n < 1 || n > MaxLimit { + return 0, ErrInvalidLimit + } + return int(n), nil +} diff --git a/internal/platform/pagination/cursor_test.go b/internal/platform/pagination/cursor_test.go new file mode 100644 index 00000000..5c93a17c --- /dev/null +++ b/internal/platform/pagination/cursor_test.go @@ -0,0 +1,71 @@ +package pagination_test + +import ( + "errors" + "strings" + "testing" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func TestCursorRoundTrip(t *testing.T) { + scope, last := ident.New(), ident.New() + encoded := pagination.EncodeCursor(scope, last) + if encoded == "" || strings.ContainsAny(encoded, "+/=") { + t.Fatalf("cursor must be non-empty URL-safe base64 without padding, got %q", encoded) + } + got, err := pagination.DecodeCursor(encoded, scope) + if err != nil { + t.Fatalf("DecodeCursor error: %v", err) + } + if got != last { + t.Fatalf("decoded %s, want %s", got, last) + } +} + +func TestDecodeCursorRejectsForeignScope(t *testing.T) { + encoded := pagination.EncodeCursor(ident.New(), ident.New()) + _, err := pagination.DecodeCursor(encoded, ident.New()) + if !errors.Is(err, pagination.ErrInvalidCursor) { + t.Fatalf("err = %v, want ErrInvalidCursor", err) + } +} + +func TestDecodeCursorRejectsGarbage(t *testing.T) { + scope := ident.New() + valid := pagination.EncodeCursor(scope, ident.New()) + for _, raw := range []string{"", "***", "abc", valid[:len(valid)-2], valid + "AA", strings.ToUpper(valid)} { + if _, err := pagination.DecodeCursor(raw, scope); !errors.Is(err, pagination.ErrInvalidCursor) { + t.Errorf("DecodeCursor(%q) err = %v, want ErrInvalidCursor", raw, err) + } + } +} + +func TestParseLimit(t *testing.T) { + cases := []struct { + raw string + want int + ok bool + }{ + {"", 50, true}, + {"1", 1, true}, + {"100", 100, true}, + {"0", 0, false}, + {"101", 0, false}, + {"-1", 0, false}, + {"+5", 0, false}, + {"5.0", 0, false}, + {"abc", 0, false}, + {" 5", 0, false}, + } + for _, tc := range cases { + got, err := pagination.ParseLimit(tc.raw) + if tc.ok && (err != nil || got != tc.want) { + t.Errorf("ParseLimit(%q) = %d, %v; want %d", tc.raw, got, err, tc.want) + } + if !tc.ok && !errors.Is(err, pagination.ErrInvalidLimit) { + t.Errorf("ParseLimit(%q) err = %v, want ErrInvalidLimit", tc.raw, err) + } + } +} diff --git a/internal/platform/pagination/page.go b/internal/platform/pagination/page.go new file mode 100644 index 00000000..f905be82 --- /dev/null +++ b/internal/platform/pagination/page.go @@ -0,0 +1,41 @@ +package pagination + +import "github.com/google/uuid" + +// Page is the service-level page: the items and the id to continue from, +// nil when the listing is exhausted. It carries the raw keyset id so that any +// transport can encode the continuation its own way. +type Page[T any] struct { + Items []T + Next *uuid.UUID +} + +// PageResponse is the wire form of a page with the continuation already +// encoded as an opaque cursor. +type PageResponse[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Cut turns a slice over-fetched by one item into a page of at most limit +// items; the extra item only proves that a next page exists. +func Cut[T any](items []T, limit int, id func(T) uuid.UUID) Page[T] { + if len(items) <= limit { + return Page[T]{Items: items} + } + items = items[:limit] + next := id(items[limit-1]) + return Page[T]{Items: items, Next: &next} +} + +func ToResponse[D, R any](p Page[D], scope uuid.UUID, convert func(D) R) PageResponse[R] { + out := PageResponse[R]{Items: make([]R, 0, len(p.Items))} + for _, item := range p.Items { + out.Items = append(out.Items, convert(item)) + } + if p.Next != nil { + cursor := EncodeCursor(scope, *p.Next) + out.NextCursor = &cursor + } + return out +} diff --git a/internal/platform/pagination/page_test.go b/internal/platform/pagination/page_test.go new file mode 100644 index 00000000..6c3b0272 --- /dev/null +++ b/internal/platform/pagination/page_test.go @@ -0,0 +1,60 @@ +package pagination_test + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func self(id uuid.UUID) uuid.UUID { return id } + +func TestCutTrimsOverfetchAndDerivesNext(t *testing.T) { + ids := []uuid.UUID{ident.New(), ident.New(), ident.New()} + + fewer := pagination.Cut(ids[:2], 5, self) + if len(fewer.Items) != 2 || fewer.Next != nil { + t.Fatalf("fewer than limit: %+v", fewer) + } + exact := pagination.Cut(ids[:2], 2, self) + if len(exact.Items) != 2 || exact.Next != nil { + t.Fatalf("exactly limit must not continue: %+v", exact) + } + over := pagination.Cut(ids, 2, self) + if len(over.Items) != 2 || over.Next == nil || *over.Next != ids[1] { + t.Fatalf("over-fetched page must trim and continue from the last kept id: %+v", over) + } + if over.Items[0] != ids[0] || over.Items[1] != ids[1] { + t.Fatalf("order changed: %v", over.Items) + } +} + +func TestToResponseEncodesCursorWithinScope(t *testing.T) { + scope, last := ident.New(), ident.New() + page := pagination.Page[uuid.UUID]{Items: []uuid.UUID{last}, Next: &last} + res := pagination.ToResponse(page, scope, func(id uuid.UUID) string { return id.String() }) + if len(res.Items) != 1 || res.Items[0] != last.String() { + t.Fatalf("items = %v", res.Items) + } + if res.NextCursor == nil { + t.Fatal("next cursor missing") + } + got, err := pagination.DecodeCursor(*res.NextCursor, scope) + if err != nil || got != last { + t.Fatalf("cursor decodes to %s, %v; want %s", got, err, last) + } +} + +func TestToResponseSerializesEmptyPageAsContractRequires(t *testing.T) { + res := pagination.ToResponse(pagination.Page[uuid.UUID]{}, ident.New(), func(id uuid.UUID) string { return id.String() }) + raw, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + if string(raw) != `{"items":[],"next_cursor":null}` { + t.Fatalf("empty page = %s", raw) + } +} diff --git a/internal/platform/password/password.go b/internal/platform/password/password.go new file mode 100644 index 00000000..7c22fdde --- /dev/null +++ b/internal/platform/password/password.go @@ -0,0 +1,74 @@ +// Package password hashes user passwords with Argon2id. +// +// A password has little entropy, so its hash must be slow and memory-hard to +// make offline guessing expensive. Session tokens are the opposite case: they +// are random 256-bit values, and a fast digest is enough for them. +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +// OWASP baseline for Argon2id. The parameters are written into every hash, +// so raising them later does not invalidate stored hashes. +const ( + memoryKiB = 19 * 1024 + iterations = 2 + threads = 1 + saltLen = 16 + keyLen = 32 +) + +var ErrMalformed = errors.New("malformed password hash") + +var b64 = base64.RawStdEncoding + +// Hash returns the PHC string $argon2id$v=19$m=…,t=…,p=…$$. +func Hash(plain string) string { + salt := make([]byte, saltLen) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte(plain), salt, iterations, memoryKiB, threads, keyLen) + return fmt.Sprintf("$argon2id$v=%d$%s$%s$%s", + argon2.Version, params(memoryKiB, iterations, threads), b64.EncodeToString(salt), b64.EncodeToString(key)) +} + +// Verify reports whether plain matches encoded. An error means the stored +// hash itself is unreadable, not that the password is wrong. +func Verify(encoded, plain string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" { + return false, ErrMalformed + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrMalformed + } + var memory, time uint32 + var parallelism uint8 + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, ¶llelism); err != nil || + memory == 0 || time == 0 || parallelism == 0 { + return false, ErrMalformed + } + salt, err := b64.DecodeString(parts[4]) + if err != nil || len(salt) == 0 { + return false, ErrMalformed + } + key, err := b64.DecodeString(parts[5]) + if err != nil || len(key) == 0 { + return false, ErrMalformed + } + got := argon2.IDKey([]byte(plain), salt, time, memory, parallelism, uint32(len(key))) + return subtle.ConstantTimeCompare(got, key) == 1, nil +} + +func params(memory, time uint32, parallelism uint8) string { + return fmt.Sprintf("m=%d,t=%d,p=%d", memory, time, parallelism) +} diff --git a/internal/platform/password/password_test.go b/internal/platform/password/password_test.go new file mode 100644 index 00000000..7c518ad1 --- /dev/null +++ b/internal/platform/password/password_test.go @@ -0,0 +1,78 @@ +package password_test + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "strings" + "testing" + + "golang.org/x/crypto/argon2" + + "messenger/internal/platform/password" +) + +func TestHashVerifiesOnlyTheOriginalPassword(t *testing.T) { + for _, plain := range []string{ + "correct horse battery staple", + "с:двоеточием:и кириллицей", + "emoji 👋👋👋 and : colon", + } { + encoded := password.Hash(plain) + ok, err := password.Verify(encoded, plain) + if err != nil || !ok { + t.Fatalf("Verify(%q) = %v, %v; want true", plain, ok, err) + } + ok, err = password.Verify(encoded, plain+"x") + if err != nil || ok { + t.Fatalf("Verify(wrong) = %v, %v; want false", ok, err) + } + } +} + +func TestHashIsSaltedPHCString(t *testing.T) { + a, b := password.Hash("same password"), password.Hash("same password") + if a == b { + t.Fatal("two hashes of one password are equal: salt is missing") + } + if !strings.HasPrefix(a, "$argon2id$v=19$m=19456,t=2,p=1$") { + t.Fatalf("unexpected encoding %q", a) + } + if strings.Contains(a, "same password") { + t.Fatal("hash contains the plaintext") + } +} + +// A hash made with other parameters must still verify: parameters travel in +// the encoded string, so they can be raised without invalidating old hashes. +func TestVerifyUsesParametersFromTheEncoding(t *testing.T) { + salt := make([]byte, 16) + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte("legacy password"), salt, 1, 8*1024, 2, 32) + b64 := base64.RawStdEncoding + encoded := fmt.Sprintf("$argon2id$v=19$m=8192,t=1,p=2$%s$%s", b64.EncodeToString(salt), b64.EncodeToString(key)) + + ok, err := password.Verify(encoded, "legacy password") + if err != nil || !ok { + t.Fatalf("Verify = %v, %v; want true", ok, err) + } +} + +func TestVerifyRejectsMalformedEncoding(t *testing.T) { + valid := password.Hash("whatever password") + parts := strings.Split(valid, "$") + for name, encoded := range map[string]string{ + "empty": "", + "wrong algo": strings.Replace(valid, "argon2id", "argon2i", 1), + "wrong version": strings.Replace(valid, "v=19", "v=16", 1), + "bad params": strings.Replace(valid, "m=19456,t=2,p=1", "m=x,t=2,p=1", 1), + "zero params": strings.Replace(valid, "m=19456,t=2,p=1", "m=19456,t=0,p=1", 1), + "bad salt": strings.Join([]string{"", parts[1], parts[2], parts[3], "!!!", parts[5]}, "$"), + "bad key": strings.Join([]string{"", parts[1], parts[2], parts[3], parts[4], "!!!"}, "$"), + "missing fields": strings.Join(parts[:5], "$"), + } { + if _, err := password.Verify(encoded, "whatever password"); err == nil { + t.Errorf("%s: Verify accepted %q", name, encoded) + } + } +} diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go new file mode 100644 index 00000000..616caf62 --- /dev/null +++ b/internal/platform/postgres/postgres.go @@ -0,0 +1,145 @@ +// Package postgres opens the connection pool and applies the schema. +// +// Pool size and server-side timeouts are constants rather than settings: they +// bound how long one request can hold a connection, and the request deadline +// of the API is chosen against them. +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/lock" + + "messenger/internal/platform/apperr" + "messenger/migrations" +) + +const ( + maxConns = 10 + connectTimeout = time.Second + statementTimeout = 5 * time.Second + idleInTxTimeout = 10 * time.Second +) + +const codeUniqueViolation = "23505" + +func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("parse database url: %w", err) + } + cfg.MaxConns = maxConns + cfg.ConnConfig.ConnectTimeout = connectTimeout + // Unlike net.Dialer, pgx resolves the host before ConnectTimeout starts. + // Docker DNS takes seconds to give up on a stopped container: unbounded, + // every request would wait out its deadline and log only "context + // deadline exceeded" instead of the failed lookup. + cfg.ConnConfig.LookupFunc = func(ctx context.Context, host string) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + return net.DefaultResolver.LookupHost(ctx, host) + } + cfg.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(statementTimeout.Milliseconds(), 10) + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = strconv.FormatInt(idleInTxTimeout.Milliseconds(), 10) + // The application keeps every time in UTC; pgx would otherwise scan + // timestamptz into time.Local. + cfg.AfterConnect = func(_ context.Context, conn *pgx.Conn) error { + conn.TypeMap().RegisterType(&pgtype.Type{ + Name: "timestamptz", + OID: pgtype.TimestamptzOID, + Codec: &pgtype.TimestamptzCodec{ScanLocation: time.UTC}, + }) + return nil + } + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("create pool: %w", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + return pool, nil +} + +// Migrate applies pending migrations and returns the versions it applied. A +// session advisory lock serialises concurrent starts, so several API +// replicas may call it at once; applied versions are recorded, which makes a +// repeated run a no-op. +func Migrate(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) { + db := stdlib.OpenDBFromPool(pool) + defer db.Close() + locker, err := lock.NewPostgresSessionLocker() + if err != nil { + return nil, fmt.Errorf("migration lock: %w", err) + } + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.FS, goose.WithSessionLocker(locker)) + if err != nil { + return nil, fmt.Errorf("load migrations: %w", err) + } + results, err := provider.Up(ctx) + if err != nil { + return nil, fmt.Errorf("apply migrations: %w", err) + } + applied := make([]int64, 0, len(results)) + for _, r := range results { + applied = append(applied, r.Source.Version) + } + return applied, nil +} + +// Classify marks a failure to reach the database as apperr.KindUnavailable, +// which the API answers with 503 instead of 500; any other error is returned +// unchanged. Unreachable means: the connection could not be made or broke, +// the request deadline passed, or the server refused work (PostgreSQL error +// classes 08 connection exception, 53 insufficient resources, 57 operator +// intervention such as shutdown or statement timeout). +func Classify(err error) error { + if err == nil || !unreachable(err) { + return err + } + return apperr.Wrap(apperr.KindUnavailable, "database unavailable", err) +} + +func unreachable(err error) bool { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + switch class := pgErr.Code[:min(2, len(pgErr.Code))]; class { + case "08", "53", "57": + return true + } + return false + } + if _, ok := errors.AsType[*pgconn.ConnectError](err); ok { + return true + } + if _, ok := errors.AsType[net.Error](err); ok { + return true + } + return errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, net.ErrClosed) || + pgconn.Timeout(err) || + pgconn.SafeToRetry(err) +} + +// IsUniqueViolation reports whether err is a unique violation of the named +// constraint. +func IsUniqueViolation(err error, constraint string) bool { + pgErr, ok := errors.AsType[*pgconn.PgError](err) + return ok && pgErr.Code == codeUniqueViolation && pgErr.ConstraintName == constraint +} diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go new file mode 100644 index 00000000..c3a14873 --- /dev/null +++ b/internal/platform/postgres/postgres_test.go @@ -0,0 +1,195 @@ +package postgres_test + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" + "messenger/internal/testenv" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +func TestPoolScansTimestamptzInUTC(t *testing.T) { + var got time.Time + if err := db.Pool.QueryRow(context.Background(), `SELECT now()`).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Location() != time.UTC { + t.Fatalf("timestamptz scanned in %v, want UTC", got.Location()) + } +} + +func TestMigrateIsRepeatable(t *testing.T) { + ctx := context.Background() + applied, err := postgres.Migrate(ctx, db.Pool) + if err != nil { + t.Fatalf("second migrate: %v", err) + } + if len(applied) != 0 { + t.Fatalf("second migrate applied %v, want nothing", applied) + } + var version int64 + if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil { + t.Fatal(err) + } + if version != 1 { + t.Fatalf("schema version = %d, want 1", version) + } + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + var exists bool + if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("table %s is missing", table) + } + } +} + +func TestPoolAppliesLimitsAndTimeouts(t *testing.T) { + if got := db.Pool.Config().MaxConns; got != 10 { + t.Fatalf("MaxConns = %d, want 10", got) + } + if got := db.Pool.Config().ConnConfig.ConnectTimeout; got != time.Second { + t.Fatalf("ConnectTimeout = %v, want 1s", got) + } + ctx := context.Background() + for setting, want := range map[string]string{ + "statement_timeout": "5s", + "idle_in_transaction_session_timeout": "10s", + } { + var got string + if err := db.Pool.QueryRow(ctx, "SHOW "+setting).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("%s = %q, want %q", setting, got, want) + } + } +} + +func TestResetEmptiesEveryTableButKeepsSchemaVersion(t *testing.T) { + ctx := context.Background() + _, err := db.Pool.Exec(ctx, ` + WITH u AS ( + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000001', 'reset_check', 'Reset', 'x', now()) + RETURNING id + ), c AS ( + INSERT INTO chats (id, title, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000002', 'Reset', now()) + RETURNING id + ), m AS ( + INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM c, u + ) + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT '0190f1d2-0000-7000-8000-000000000003', c.id, u.id, 'hi', now() FROM c, u`) + if err != nil { + t.Fatal(err) + } + + db.Reset(t) + + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + var n int + if err := db.Pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%s has %d rows after Reset", table, n) + } + } + var versions int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM goose_db_version`).Scan(&versions); err != nil { + t.Fatal(err) + } + if versions == 0 { + t.Fatal("Reset wiped the schema version table") + } +} + +func TestIsUniqueViolation(t *testing.T) { + db.Reset(t) + ctx := context.Background() + insert := `INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, 'taken', 'Taken', 'x', now())` + if _, err := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000011"); err != nil { + t.Fatal(err) + } + _, dup := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000012") + if dup == nil { + t.Fatal("duplicate username was accepted") + } + + if !postgres.IsUniqueViolation(dup, "users_username_key") { + t.Fatalf("not recognised as a username violation: %v", dup) + } + if postgres.IsUniqueViolation(dup, "some_other_key") { + t.Fatal("matched a different constraint") + } + if postgres.IsUniqueViolation(fmt.Errorf("wrapped: %w", dup), "users_username_key") == false { + t.Fatal("wrapped error not recognised") + } + if postgres.IsUniqueViolation(errors.New("plain"), "users_username_key") || postgres.IsUniqueViolation(nil, "users_username_key") { + t.Fatal("non-database error recognised as a violation") + } +} + +func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { + unavailable := map[string]error{ + "admin shutdown": &pgconn.PgError{Code: "57P01"}, + "statement timeout": &pgconn.PgError{Code: "57014"}, + "connection failure": &pgconn.PgError{Code: "08006"}, + "too many connections": &pgconn.PgError{Code: "53300"}, + "deadline": fmt.Errorf("query: %w", context.DeadlineExceeded), + "network": &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")}, + "unexpected eof": fmt.Errorf("read: %w", io.ErrUnexpectedEOF), + } + for name, err := range unavailable { + got := postgres.Classify(err) + if apperr.KindOf(got) != apperr.KindUnavailable || !errors.Is(got, err) { + t.Errorf("%s: Classify = %v (kind %v), want KindUnavailable keeping the cause", name, got, apperr.KindOf(got)) + } + } + for name, err := range map[string]error{ + "unique violation": &pgconn.PgError{Code: "23505"}, + "plain": errors.New("boom"), + "nil": nil, + } { + if got := postgres.Classify(err); got != err { + t.Errorf("%s: Classify = %v, want the error unchanged", name, got) + } + } +} + +func TestClassifyRealConnectFailure(t *testing.T) { + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1") + if err == nil { + t.Fatal("NewPool to a closed port must fail") + } + if apperr.KindOf(postgres.Classify(err)) != apperr.KindUnavailable { + t.Fatalf("connect failure classified as %v: %v", apperr.KindOf(postgres.Classify(err)), err) + } +} diff --git a/internal/platform/valkey/valkey.go b/internal/platform/valkey/valkey.go new file mode 100644 index 00000000..70c5ba5d --- /dev/null +++ b/internal/platform/valkey/valkey.go @@ -0,0 +1,49 @@ +// Package valkey connects to the Valkey session store. +// +// Timeouts are constants: every authenticated request reads a session, so +// they bound how long a request can wait when the store is slow. Command +// retries are disabled and a connection is dialled once for the same reason: +// a request that cannot reach the store fails fast with 503 instead of +// waiting through a backoff. On the compose stand Docker DNS takes seconds +// to give up on a stopped container, so one dial attempt is already bounded +// only by dialTimeout. +package valkey + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + dialTimeout = time.Second + ioTimeout = time.Second + poolSize = 10 + noRetries = -1 + singleAttempt = 1 +) + +// NewClient connects to url (redis:// scheme) and pings the server. Errors +// never include the URL, which carries the password. +func NewClient(ctx context.Context, url string) (*redis.Client, error) { + opts, err := redis.ParseURL(url) + if err != nil { + return nil, errors.New("session store url is invalid") + } + opts.DialTimeout = dialTimeout + opts.ReadTimeout = ioTimeout + opts.WriteTimeout = ioTimeout + opts.PoolSize = poolSize + opts.MaxRetries = noRetries + opts.DialerRetries = singleAttempt + + client := redis.NewClient(opts) + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, fmt.Errorf("ping session store at %s: %w", opts.Addr, err) + } + return client, nil +} diff --git a/internal/platform/valkey/valkey_test.go b/internal/platform/valkey/valkey_test.go new file mode 100644 index 00000000..efb43ebf --- /dev/null +++ b/internal/platform/valkey/valkey_test.go @@ -0,0 +1,57 @@ +package valkey_test + +import ( + "context" + "strings" + "testing" + "time" + + "messenger/internal/platform/valkey" + "messenger/internal/testenv" +) + +func TestNewClientPingsAndBoundsWaiting(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + client, err := valkey.NewClient(ctx, kv.URL) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer client.Close() + opts := client.Options() + if opts.DialTimeout != time.Second || opts.ReadTimeout != time.Second || opts.WriteTimeout != time.Second { + t.Fatalf("timeouts = %v/%v/%v, want 1s each", opts.DialTimeout, opts.ReadTimeout, opts.WriteTimeout) + } + // go-redis dials up to five times by default; with Docker DNS taking + // seconds to give up on a stopped container that alone would exceed the + // request deadline. + if opts.DialerRetries != 1 { + t.Fatalf("DialerRetries = %d, want a single dial attempt", opts.DialerRetries) + } + // go-redis stores the requested -1 ("no retries") as 0 after init; its + // default would be 3. + if opts.PoolSize != 10 || opts.MaxRetries != 0 { + t.Fatalf("PoolSize = %d, MaxRetries = %d; want 10 and 0 (fail fast)", opts.PoolSize, opts.MaxRetries) + } +} + +func TestNewClientErrorsNeverEchoTheURL(t *testing.T) { + ctx := context.Background() + for name, url := range map[string]string{ + "malformed": "redis://:s3cret@host:port/x", + "unreachable": "redis://:s3cret@127.0.0.1:1/0", + } { + _, err := valkey.NewClient(ctx, url) + if err == nil { + t.Fatalf("%s: want error", name) + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("%s: error leaks the password: %v", name, err) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 00000000..8a70e0fe --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,60 @@ +// Package server runs the HTTP listener with the lifecycle the lab +// requires: readiness drops first, the listener stops accepting, in-flight +// requests drain within a bounded timeout, then the process exits. +package server + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "time" +) + +// Readiness is switched off when shutdown starts. +type Readiness interface { + SetReady(ready bool) +} + +// Run serves until ctx is cancelled, then shuts down within shutdownTimeout. +func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, shutdownTimeout time.Duration) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 64 * 1024, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + + served := make(chan error, 1) + go func() { + err := srv.Serve(ln) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + served <- err + }() + log.Info("listening", "addr", ln.Addr().String()) + + select { + case err := <-served: + return err + case <-ctx.Done(): + } + + log.Info("shutdown started", "timeout", shutdownTimeout.String()) + ready.SetReady(false) + drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := srv.Shutdown(drain); err != nil { + log.Error("shutdown forced", "error", err.Error()) + _ = srv.Close() + return err + } + <-served + log.Info("shutdown completed") + return nil +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 00000000..040f172a --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,102 @@ +package server_test + +import ( + "context" + "io" + "log/slog" + "net" + "net/http" + "testing" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/health" + "messenger/internal/platform/httpx" + "messenger/internal/server" +) + +// newHandler builds the smallest handler with probes; the lifecycle does not +// depend on the business modules or their stores. +func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { + e := httpx.NewEngine(log, 1024) + probes := health.New() + probes.Register(e) + return e, probes +} + +func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop within 10s after cancel") + } + if _, err := http.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after shutdown") + } +} + +func TestReadinessDropsBeforeListenerCloses(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + // Hold one request open so shutdown has something to drain. + slow := make(chan struct{}) + go func() { + resp, err := http.Get(base + "/health/live") + if err == nil { + resp.Body.Close() + } + close(slow) + }() + + cancel() + <-done + <-slow + if !probes.IsReady() { + return + } + t.Fatal("readiness must be switched off during shutdown") +} + +func waitFor(t *testing.T, url string, status int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + if resp.StatusCode == status { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not return %d in time", url, status) +} diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go new file mode 100644 index 00000000..980d191d --- /dev/null +++ b/internal/testenv/testenv.go @@ -0,0 +1,181 @@ +// Package testenv starts disposable PostgreSQL and Valkey containers for tests +// that exercise the real stores. Only _test files import it, so it never +// reaches the server binary. The images match compose.yaml. +package testenv + +import ( + "context" + "fmt" + "os" + "runtime" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcvalkey "github.com/testcontainers/testcontainers-go/modules/valkey" + + "messenger/internal/platform/postgres" + "messenger/internal/platform/valkey" +) + +const ( + PostgresImage = "postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873" + ValkeyImage = "valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11" +) + +// useDockerDesktopPipe names the Docker Desktop engine pipe explicitly on +// Windows. Without DOCKER_HOST, testcontainers probes the pipe with a file +// stat, which fails while the pipe is busy; with several test packages +// starting containers at once that probe fails spuriously and the run panics. +// Dialling the named host instead waits for a free pipe instance. +var useDockerDesktopPipe = sync.OnceFunc(func() { + if runtime.GOOS == "windows" && os.Getenv("DOCKER_HOST") == "" { + _ = os.Setenv("DOCKER_HOST", "npipe:////./pipe/docker_engine") + } +}) + +// DB is a migrated PostgreSQL instance. +type DB struct { + Pool *pgxpool.Pool + DSN string + container *tcpostgres.PostgresContainer +} + +func StartPostgres(ctx context.Context) (*DB, error) { + useDockerDesktopPipe() + c, err := tcpostgres.Run(ctx, PostgresImage, + tcpostgres.WithDatabase("messenger"), + tcpostgres.WithUsername("messenger"), + tcpostgres.WithPassword("messenger"), + tcpostgres.BasicWaitStrategies(), + ) + if err != nil { + return nil, fmt.Errorf("start postgres: %w", err) + } + db := &DB{container: c} + db.DSN, err = c.ConnectionString(ctx, "sslmode=disable") + if err != nil { + db.Close() + return nil, fmt.Errorf("postgres address: %w", err) + } + db.Pool, err = postgres.NewPool(ctx, db.DSN) + if err != nil { + db.Close() + return nil, err + } + if _, err := postgres.Migrate(ctx, db.Pool); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// Reset empties every application table, so each test starts from an empty +// schema. The table list comes from the catalog: a table added by a later +// migration cannot be forgotten here. +func (d *DB) Reset(t testing.TB) { + t.Helper() + ctx := context.Background() + var tables string + err := d.Pool.QueryRow(ctx, ` + SELECT coalesce(string_agg(format('%I.%I', schemaname, tablename), ', '), '') + FROM pg_tables + WHERE schemaname = 'public' AND tablename <> 'goose_db_version'`).Scan(&tables) + if err != nil { + t.Fatalf("list tables: %v", err) + } + if tables == "" { + return + } + if _, err := d.Pool.Exec(ctx, "TRUNCATE "+tables+" CASCADE"); err != nil { + t.Fatalf("truncate: %v", err) + } +} + +// Stop stops the container, as when the database goes down. +func (d *DB) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return d.container.Stop(ctx, &timeout) +} + +// Pause freezes the server: connections stay open but nothing answers, as +// with a hung database. +func (d *DB) Pause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerPause(ctx, d.container.GetContainerID()) + }) +} + +func (d *DB) Unpause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerUnpause(ctx, d.container.GetContainerID()) + }) +} + +func withDocker(ctx context.Context, fn func(cli *testcontainers.DockerClient) error) error { + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + return err + } + defer cli.Close() + return fn(cli) +} + +func (d *DB) Close() { + if d.Pool != nil { + d.Pool.Close() + } + _ = testcontainers.TerminateContainer(d.container) +} + +// KV is a Valkey instance without persistence or password. +type KV struct { + Client *redis.Client + URL string + container *tcvalkey.ValkeyContainer +} + +func StartValkey(ctx context.Context) (*KV, error) { + useDockerDesktopPipe() + c, err := tcvalkey.Run(ctx, ValkeyImage) + if err != nil { + return nil, fmt.Errorf("start valkey: %w", err) + } + kv := &KV{container: c} + kv.URL, err = c.ConnectionString(ctx) + if err != nil { + kv.Close() + return nil, fmt.Errorf("valkey address: %w", err) + } + // The production client, so outage tests see its fail-fast settings. + kv.Client, err = valkey.NewClient(ctx, kv.URL) + if err != nil { + kv.Close() + return nil, err + } + return kv, nil +} + +func (k *KV) Reset(t testing.TB) { + t.Helper() + if err := k.Client.FlushDB(context.Background()).Err(); err != nil { + t.Fatalf("flushdb: %v", err) + } +} + +// Stop stops the container, as when the session store goes down. +func (k *KV) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return k.container.Stop(ctx, &timeout) +} + +func (k *KV) Close() { + if k.Client != nil { + _ = k.Client.Close() + } + _ = testcontainers.TerminateContainer(k.container) +} diff --git a/internal/testenv/testenv_test.go b/internal/testenv/testenv_test.go new file mode 100644 index 00000000..6488bded --- /dev/null +++ b/internal/testenv/testenv_test.go @@ -0,0 +1,29 @@ +package testenv_test + +import ( + "context" + "testing" + + "messenger/internal/testenv" +) + +func TestValkeyStartsAndResetFlushesKeys(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + if err := kv.Client.Set(ctx, "probe", "1", 0).Err(); err != nil { + t.Fatal(err) + } + kv.Reset(t) + n, err := kv.Client.DBSize(ctx).Result() + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%d keys left after Reset", n) + } +} diff --git a/internal/user/handler.go b/internal/user/handler.go new file mode 100644 index 00000000..a5f091af --- /dev/null +++ b/internal/user/handler.go @@ -0,0 +1,70 @@ +package user + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/httpx" +) + +type Handler struct { + users *Service +} + +func NewHandler(users *Service) *Handler { + return &Handler{users: users} +} + +// Register mounts registration on the public router and the current-user +// endpoint on the router guarded by the authentication middleware. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/users", h.register) + protected.GET("/users/me", httpx.WithUser(h.me)) +} + +type createRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type Response struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +func ToResponse(u User) Response { + return Response{ + ID: u.ID.String(), + Username: u.Username, + DisplayName: u.DisplayName, + CreatedAt: httpx.Timestamp(u.CreatedAt), + } +} + +func (h *Handler) register(c *gin.Context) { + var req createRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + u, err := h.users.Register(c.Request.Context(), req.Username, req.DisplayName, req.Password) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, ToResponse(u)) +} + +func (h *Handler) me(c *gin.Context, actor uuid.UUID) { + u, err := h.users.GetByID(c.Request.Context(), actor) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, ToResponse(u)) +} diff --git a/internal/user/main_test.go b/internal/user/main_test.go new file mode 100644 index 00000000..ba3a1ca2 --- /dev/null +++ b/internal/user/main_test.go @@ -0,0 +1,24 @@ +package user_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} diff --git a/internal/user/repository.go b/internal/user/repository.go new file mode 100644 index 00000000..eb2ac2f6 --- /dev/null +++ b/internal/user/repository.go @@ -0,0 +1,18 @@ +package user + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores a new user or returns ErrUsernameTaken; the uniqueness + // check and the insert must be atomic. + Create(ctx context.Context, user User) error + GetByID(ctx context.Context, id uuid.UUID) (User, error) + GetByUsername(ctx context.Context, username string) (User, error) + // ExistingIDs returns the subset of ids that belong to registered users, + // in no particular order, in one round trip. + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} diff --git a/internal/user/service.go b/internal/user/service.go new file mode 100644 index 00000000..708ddb9b --- /dev/null +++ b/internal/user/service.go @@ -0,0 +1,88 @@ +package user + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/password" +) + +type Service struct { + repo Repository +} + +func NewService(repo Repository) *Service { + return &Service{repo: repo} +} + +// Register validates every field before hashing, so invalid input never pays +// for the KDF. +func (s *Service) Register(ctx context.Context, username, displayName, plain string) (User, error) { + u, err := New(ident.New(), username, displayName) + if err != nil { + return User{}, err + } + if err := ValidatePassword(plain); err != nil { + return User{}, err + } + u.PasswordHash = password.Hash(plain) + if err := s.repo.Create(ctx, u); err != nil { + return User{}, err + } + return u, nil +} + +// dummyHash is verified for unknown usernames, so a failed login takes the +// same time whether or not the user exists. +var dummyHash = sync.OnceValue(func() string { + return password.Hash("timing-equaliser-password") +}) + +// Authenticate returns the user whose password matches, or +// ErrInvalidCredentials for an unknown username and a wrong password alike. +// The username comes straight from the Basic header, so one that cannot +// exist (including NUL or invalid UTF-8, which the database would reject) +// is answered without a query, but after the same KDF work. +func (s *Service) Authenticate(ctx context.Context, username, plain string) (User, error) { + if !usernamePattern.MatchString(username) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + u, err := s.repo.GetByUsername(ctx, username) + if errors.Is(err, ErrNotFound) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + if err != nil { + return User{}, fmt.Errorf("find user: %w", err) + } + ok, err := password.Verify(u.PasswordHash, plain) + if err != nil { + return User{}, fmt.Errorf("verify password of user %s: %w", u.ID, err) + } + if !ok { + return User{}, ErrInvalidCredentials + } + return u, nil +} + +func (s *Service) GetByID(ctx context.Context, id uuid.UUID) (User, error) { + return s.repo.GetByID(ctx, id) +} + +// ExistingIDs serves the chats module when it validates member lists. +func (s *Service) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + if len(ids) == 0 { + return []uuid.UUID{}, nil + } + found, err := s.repo.ExistingIDs(ctx, ids) + if err != nil { + return nil, fmt.Errorf("user lookup: %w", err) + } + return found, nil +} diff --git a/internal/user/service_test.go b/internal/user/service_test.go new file mode 100644 index 00000000..3406ddd8 --- /dev/null +++ b/internal/user/service_test.go @@ -0,0 +1,189 @@ +package user_test + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" + "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +// newService wires the service to the real store over an empty database. +func newService(t *testing.T) *user.Service { + t.Helper() + db.Reset(t) + return user.NewService(store.NewUsers(db.Pool)) +} + +func TestRegisterStoresUserWithPasswordHash(t *testing.T) { + svc := newService(t) + ctx := context.Background() + u, err := svc.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatalf("Register error: %v", err) + } + if u.ID.Version() != 7 { + t.Fatalf("id must be UUIDv7, got version %d", u.ID.Version()) + } + if !strings.HasPrefix(u.PasswordHash, "$argon2id$") || strings.Contains(u.PasswordHash, pw) { + t.Fatalf("password must be stored as an Argon2id hash, got %q", u.PasswordHash) + } + got, err := svc.GetByID(ctx, u.ID) + if err != nil { + t.Fatalf("GetByID error: %v", err) + } + if got != u { + t.Fatalf("GetByID = %+v, want %+v", got, u) + } +} + +func TestRegisterRejectsInvalidInputAndStoresNothing(t *testing.T) { + svc := newService(t) + ctx := context.Background() + cases := map[string][3]string{ + "username": {"Bad Name", "x", pw}, + "display_name": {"alice", " ", pw}, + "password": {"alice", "Алиса", "short"}, + } + for field, in := range cases { + _, err := svc.Register(ctx, in[0], in[1], in[2]) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), field) { + t.Errorf("%s: err = %v, want KindInvalid naming the field", field, err) + } + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&n); err != nil || n != 0 { + t.Fatalf("users = %d, %v; invalid input must not be stored", n, err) + } +} + +func TestRegisterReportsDuplicateAsConflict(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "A", pw); err != nil { + t.Fatal(err) + } + _, err := svc.Register(ctx, "alice", "B", pw) + if !errors.Is(err, user.ErrUsernameTaken) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("err = %v, want ErrUsernameTaken with KindConflict", err) + } +} + +// Concurrent registrations of one username race on the unique constraint: +// exactly one wins and every other gets the conflict, never a 500. +func TestConcurrentRegistrationOfOneUsernameHasOneWinner(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const attempts = 10 + errs := make(chan error, attempts) + var start, done sync.WaitGroup + start.Add(1) + for range attempts { + done.Add(1) + go func() { + defer done.Done() + start.Wait() + _, err := svc.Register(ctx, "racer", "Racer", pw) + errs <- err + }() + } + start.Done() + done.Wait() + close(errs) + + won, lost := 0, 0 + for err := range errs { + switch { + case err == nil: + won++ + case errors.Is(err, user.ErrUsernameTaken): + lost++ + default: + t.Errorf("unexpected error: %v", err) + } + } + if won != 1 || lost != attempts-1 { + t.Fatalf("won = %d, lost = %d; want 1 and %d", won, lost, attempts-1) + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users WHERE username = 'racer'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("rows = %d, %v; want exactly one", n, err) + } +} + +func TestGetByIDUnknownIsNotFound(t *testing.T) { + svc := newService(t) + _, err := svc.GetByID(context.Background(), ident.New()) + if !errors.Is(err, user.ErrNotFound) || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("err = %v, want ErrNotFound with KindNotFound", err) + } +} + +func TestExistingIDsReturnsOnlyRegistered(t *testing.T) { + svc := newService(t) + ctx := context.Background() + alice, _ := svc.Register(ctx, "alice", "A", pw) + bob, _ := svc.Register(ctx, "bob", "B", pw) + got, err := svc.ExistingIDs(ctx, []uuid.UUID{ident.New(), bob.ID, alice.ID}) + if err != nil { + t.Fatal(err) + } + set := map[uuid.UUID]bool{} + for _, id := range got { + set[id] = true + } + if len(got) != 2 || !set[alice.ID] || !set[bob.ID] { + t.Fatalf("ExistingIDs = %v, want alice and bob", got) + } + if empty, err := svc.ExistingIDs(ctx, nil); err != nil || len(empty) != 0 { + t.Fatalf("ExistingIDs(nil) = %v, %v", empty, err) + } +} + +func TestAuthenticateChecksPassword(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const colonPassword = "пароль:с:двоеточием 👋" + alice, err := svc.Register(ctx, "alice", "Алиса", colonPassword) + if err != nil { + t.Fatal(err) + } + got, err := svc.Authenticate(ctx, "alice", colonPassword) + if err != nil || got != alice { + t.Fatalf("Authenticate = %+v, %v; want alice", got, err) + } +} + +// A wrong password and an unknown username must be indistinguishable. +func TestAuthenticateFailuresLookTheSame(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "Алиса", pw); err != nil { + t.Fatal(err) + } + _, wrong := svc.Authenticate(ctx, "alice", pw+"!") + _, unknown := svc.Authenticate(ctx, "nobody", pw) + // Basic credentials are client bytes: a username the database cannot + // even compare must still be a plain credential failure. + _, nul := svc.Authenticate(ctx, "a\x00b", pw) + _, notUTF8 := svc.Authenticate(ctx, "\xff\xfe", pw) + for name, err := range map[string]error{ + "wrong password": wrong, "unknown user": unknown, "nul username": nul, "invalid utf-8 username": notUTF8, + } { + if !errors.Is(err, user.ErrInvalidCredentials) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if wrong.Error() != unknown.Error() { + t.Fatalf("messages differ: %q vs %q", wrong, unknown) + } +} diff --git a/internal/user/store/postgres.go b/internal/user/store/postgres.go new file mode 100644 index 00000000..d8974847 --- /dev/null +++ b/internal/user/store/postgres.go @@ -0,0 +1,74 @@ +// Package store holds the repository implementations of the users module. +// Errors pass through postgres.Classify, so an unreachable database surfaces +// as unavailability rather than as an internal error. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/platform/postgres" + "messenger/internal/user" +) + +type Users struct { + pool *pgxpool.Pool +} + +var _ user.Repository = (*Users)(nil) + +func NewUsers(pool *pgxpool.Pool) *Users { + return &Users{pool: pool} +} + +// Create relies on the unique constraint rather than a prior lookup, so two +// concurrent registrations of one username cannot both succeed. +func (r *Users) Create(ctx context.Context, u user.User) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES ($1, $2, $3, $4, $5)`, + u.ID, u.Username, u.DisplayName, u.PasswordHash, u.CreatedAt) + if postgres.IsUniqueViolation(err, "users_username_key") { + return user.ErrUsernameTaken + } + if err != nil { + return fmt.Errorf("insert user: %w", postgres.Classify(err)) + } + return nil +} + +func (r *Users) GetByID(ctx context.Context, id uuid.UUID) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1`, id) +} + +func (r *Users) GetByUsername(ctx context.Context, username string) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1`, username) +} + +func (r *Users) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + rows, err := r.pool.Query(ctx, `SELECT id FROM users WHERE id = ANY($1)`, ids) + if err != nil { + return nil, fmt.Errorf("select user ids: %w", postgres.Classify(err)) + } + found, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) + if err != nil { + return nil, fmt.Errorf("read user ids: %w", postgres.Classify(err)) + } + return found, nil +} + +func (r *Users) one(ctx context.Context, query string, arg any) (user.User, error) { + var u user.User + err := r.pool.QueryRow(ctx, query, arg).Scan(&u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return user.User{}, user.ErrNotFound + } + if err != nil { + return user.User{}, fmt.Errorf("select user: %w", postgres.Classify(err)) + } + return u, nil +} diff --git a/internal/user/user.go b/internal/user/user.go new file mode 100644 index 00000000..d05e1c43 --- /dev/null +++ b/internal/user/user.go @@ -0,0 +1,75 @@ +// Package user is the users module. The entity mirrors the table row and +// carries no behaviour; New only enforces input invariants and CreatedAt +// derives from the UUIDv7 id. The password itself is never kept: the service +// validates it and stores only its hash. +package user + +import ( + "fmt" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxDisplayNameCodePoints = 100 + MinPasswordCodePoints = 12 + MaxPasswordCodePoints = 128 +) + +var ( + ErrNotFound = apperr.NotFound("user not found") + ErrUsernameTaken = apperr.Conflict("username already taken") + // ErrInvalidCredentials covers an unknown username and a wrong password + // alike, so a login attempt cannot probe which usernames exist. + ErrInvalidCredentials = apperr.Unauthorized("invalid username or password") +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +func New(id uuid.UUID, username, displayName string) (User, error) { + if !usernamePattern.MatchString(username) { + return User{}, apperr.Invalid("username must match [a-z0-9_]{3,32}") + } + if strings.TrimSpace(displayName) == "" { + return User{}, apperr.Invalid("display_name must not be blank") + } + if utf8.RuneCountInString(displayName) > MaxDisplayNameCodePoints { + return User{}, apperr.Invalid(fmt.Sprintf("display_name must be at most %d characters", MaxDisplayNameCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(displayName, 0) { + return User{}, apperr.Invalid("display_name must not contain the NUL character") + } + return User{ + ID: id, + Username: username, + DisplayName: displayName, + CreatedAt: ident.TimeOf(id), + }, nil +} + +// ValidatePassword checks the contract length in Unicode code points. +func ValidatePassword(plain string) error { + if strings.TrimSpace(plain) == "" { + return apperr.Invalid("password must not be blank") + } + if n := utf8.RuneCountInString(plain); n < MinPasswordCodePoints || n > MaxPasswordCodePoints { + return apperr.Invalid(fmt.Sprintf("password must be %d to %d characters", MinPasswordCodePoints, MaxPasswordCodePoints)) + } + return nil +} diff --git a/internal/user/user_test.go b/internal/user/user_test.go new file mode 100644 index 00000000..ede9ec52 --- /dev/null +++ b/internal/user/user_test.go @@ -0,0 +1,78 @@ +package user_test + +import ( + "strings" + "testing" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" +) + +func TestNewDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + u, err := user.New(id, "alice", "Алиса") + if err != nil { + t.Fatalf("New error: %v", err) + } + if u.ID != id || u.Username != "alice" || u.DisplayName != "Алиса" { + t.Fatalf("unexpected user %+v", u) + } + if !u.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", u.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewValidatesUsername(t *testing.T) { + valid := []string{"abc", "a_1", strings.Repeat("a", 32), "user_2026"} + for _, name := range valid { + if _, err := user.New(ident.New(), name, "ok"); err != nil { + t.Errorf("username %q must be accepted: %v", name, err) + } + } + invalid := []string{"", "ab", strings.Repeat("a", 33), "Alice", "al ice", "алиса", "a-b", "a.b", " abc", "abc\n"} + for _, name := range invalid { + _, err := user.New(ident.New(), name, "ok") + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "username") { + t.Errorf("username %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} + +// The contract counts the password length in Unicode code points, not bytes. +func TestValidatePasswordCountsCodePoints(t *testing.T) { + valid := []string{ + strings.Repeat("я", 12), + strings.Repeat("👋", 128), + "pass:word:with:colons", + " spaced but long enough ", + } + for _, p := range valid { + if err := user.ValidatePassword(p); err != nil { + t.Errorf("password of %d code points must be accepted: %v", len([]rune(p)), err) + } + } + invalid := []string{"", strings.Repeat(" ", 12), strings.Repeat("\t", 20), strings.Repeat("я", 11), strings.Repeat("👋", 129)} + for _, p := range invalid { + err := user.ValidatePassword(p) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "password") { + t.Errorf("password %q must be rejected as invalid naming the field, got %v", p, err) + } + } +} + +func TestNewValidatesDisplayNameByCodePoints(t *testing.T) { + valid := []string{"A", "Алиса", strings.Repeat("я", 100), "👋 hi"} + for _, name := range valid { + if _, err := user.New(ident.New(), "alice", name); err != nil { + t.Errorf("display_name %q must be accepted: %v", name, err) + } + } + invalid := []string{"", " ", "\t\n", strings.Repeat("я", 101), "nul\x00inside"} + for _, name := range invalid { + _, err := user.New(ident.New(), "alice", name) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "display_name") { + t.Errorf("display_name %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} diff --git a/migrations/00001_init.sql b/migrations/00001_init.sql new file mode 100644 index 00000000..400b762e --- /dev/null +++ b/migrations/00001_init.sql @@ -0,0 +1,42 @@ +-- +goose Up +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL CONSTRAINT users_username_key UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +-- Lists the chats of one user in (created_at, id) order: chat ids are UUIDv7. +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +-- The key is (chat_id, id) and serves the history query directly: +-- WHERE chat_id = $1 AND id > $2 ORDER BY id LIMIT $3. Nothing looks a +-- message up by id alone, and a separate index on id would only cost writes +-- and tempt the planner into scanning every chat's messages in id order. +CREATE TABLE messages ( + chat_id uuid NOT NULL REFERENCES chats (id), + id uuid NOT NULL, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, id) +); + +-- +goose Down +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 00000000..29af9ab0 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +// Package migrations embeds the versioned SQL schema, so the server binary +// always carries the schema it was built against. +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS diff --git a/scripts/explain-history.sql b/scripts/explain-history.sql new file mode 100644 index 00000000..f455eedd --- /dev/null +++ b/scripts/explain-history.sql @@ -0,0 +1,75 @@ +-- EXPLAIN evidence for the message history query (make explain). +-- +-- Everything runs in one transaction that is rolled back: it seeds 12 000 +-- messages in one chat and 90 000 in 300 other chats, so the plan is measured +-- on a realistic volume without leaving data on the stand. The PREPARE text +-- is the statement of Messages.ListByChat in internal/chat/store/postgres.go; +-- keep them identical. +\set ON_ERROR_STOP on +\pset pager off +BEGIN; + +SELECT uuidv7() AS author \gset +SELECT uuidv7() AS big_chat \gset + +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES (:'author', 'explain_' || substr(md5(random()::text), 1, 12), 'EXPLAIN', 'not-a-password-hash', + uuid_extract_timestamp(:'author')); + +CREATE TEMP TABLE noise_chats ON COMMIT DROP AS +SELECT uuidv7() AS id FROM generate_series(1, 300); + +INSERT INTO chats (id, title, created_at) +SELECT :'big_chat'::uuid, 'explain: big chat', uuid_extract_timestamp(:'big_chat') +UNION ALL +SELECT id, 'explain: other chat', uuid_extract_timestamp(id) FROM noise_chats; + +INSERT INTO chat_members (chat_id, user_id) +SELECT id, :'author' FROM chats WHERE title LIKE 'explain:%'; + +-- Rows are inserted ordered by n, so the messages of all chats interleave in +-- the heap as they would under real traffic; ids stay increasing per chat. +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT id, chat_id, :'author', 'message ' || n, uuid_extract_timestamp(id) +FROM ( + SELECT uuidv7() AS id, chat_id, n + FROM ( + SELECT :'big_chat'::uuid AS chat_id, n FROM generate_series(1, 12000) AS n + UNION ALL + SELECT c.id, n FROM noise_chats AS c, generate_series(1, 300) AS n + ) AS planned + ORDER BY n +) AS generated; + +ANALYZE users, chats, chat_members, messages; + +SELECT (SELECT count(*) FROM messages WHERE chat_id = :'big_chat') AS big_chat_messages, + (SELECT count(*) FROM messages) AS all_messages; + +PREPARE history(uuid, uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 + AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY id +LIMIT $3; + +\echo +\echo == First page: no cursor, limit 50 (+1 row that tells whether a next page exists) +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', NULL, 51); + +SELECT id AS deep_cursor FROM messages WHERE chat_id = :'big_chat' ORDER BY id OFFSET 10000 LIMIT 1 \gset +\echo +\echo == Deep page: cursor after the 10 001st message of the chat +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', :'deep_cursor', 51); + +\echo +\echo == For contrast, OFFSET pagination of the same page reads every skipped row +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = :'big_chat' +ORDER BY id +OFFSET 10001 LIMIT 51; + +ROLLBACK; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..ae7df48b --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Creates the local credentials of the compose stand in ./secrets, which git +# ignores. Existing secrets are kept: PostgreSQL applies its password only +# when the volume is first initialised, so a new password would lock the API +# out of an existing database. To start over: make destroy, then delete +# ./secrets and run this again. +set -eu +cd "$(dirname "$0")/.." + +files="postgres_password database_url valkey_auth.conf session_store_url" +present=0 +for f in $files; do + if [ -e "secrets/$f" ]; then + present=$((present + 1)) + fi +done +if [ "$present" -eq 4 ]; then + echo "secrets already exist in ./secrets, kept" + exit 0 +fi +if [ "$present" -ne 0 ]; then + echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2 + exit 1 +fi + +# 24 random bytes as hex: URL-safe without escaping. +random() { + od -An -tx1 -N24 /dev/urandom | tr -d ' \n' +} +pg=$(random) +kv=$(random) + +# The directory is private; the files stay readable for the unprivileged +# users inside the containers, which see them through bind mounts. +umask 022 +mkdir -p secrets +chmod 700 secrets +printf '%s\n' "$pg" > secrets/postgres_password +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url +printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf +printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url +echo "created ./secrets" diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..5f86f138 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/restart.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0restart.sh" +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..11591f9c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Restart hook of `make test-persistence` (tests/resilience.py): stops the +# API, PostgreSQL and Valkey gracefully and starts the same containers again. +# Volumes and data are kept and nothing is re-seeded. The test hides the +# output, so failures surface through the exit code only. +set -eu +cd "$(dirname "$0")/.." +docker compose stop +docker compose up -d --wait diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go new file mode 100644 index 00000000..5f915761 --- /dev/null +++ b/tools/concurrency/client.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// client is a minimal typed HTTP client for the messenger API. +type client struct { + base string + http *http.Client +} + +func newClient(baseURL string) *client { + return &client{base: baseURL, http: &http.Client{Timeout: 10 * time.Second}} +} + +func (c *client) post(path, token string, body, out any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+path, bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, token, http.StatusCreated, out) +} + +// login opens a session with Basic credentials and returns its bearer token. +func (c *client) login(username, password string) (string, error) { + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + req.SetBasicAuth(username, password) + var session struct { + Token string `json:"token"` + } + if err := c.do(req, "", http.StatusCreated, &session); err != nil { + return "", err + } + return session.Token, nil +} + +// get requests the largest page the contract allows, continuing from cursor +// when it is set. +func (c *client) get(path, token, cursor string, out any) error { + query := url.Values{"limit": {"100"}} + if cursor != "" { + query.Set("cursor", cursor) + } + req, err := http.NewRequest(http.MethodGet, c.base+path+"?"+query.Encode(), nil) + if err != nil { + return err + } + return c.do(req, token, http.StatusOK, out) +} + +func (c *client) do(req *http.Request, token string, want int, out any) error { + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return err + } + if resp.StatusCode != want { + return fmt.Errorf("%s %s: status %d, want %d: %s", req.Method, req.URL.Path, resp.StatusCode, want, payload) + } + return json.Unmarshal(payload, out) +} diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go new file mode 100644 index 00000000..0ed1e54e --- /dev/null +++ b/tools/concurrency/main.go @@ -0,0 +1,227 @@ +// Command concurrency is the black-box race check from the grading rubric: +// N clients send M messages each into one chat at the same time, then the +// history is paged and must contain exactly N*M unique messages in id order. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/user" +) + +// Report is the machine readable outcome saved as evidence. +type Report struct { + BaseURL string `json:"base_url"` + StartedAt time.Time `json:"started_at"` + Clients int `json:"clients"` + PerClient int `json:"per_client"` + Expected int `json:"expected_messages"` + Created int `json:"created_201"` + Failed int `json:"failed_posts"` + Listed int `json:"listed_messages"` + Unique int `json:"unique_ids"` + Missing int `json:"created_but_not_listed"` + Ordered bool `json:"history_ordered_by_id"` + Pages int `json:"pages_read"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + Passed bool `json:"passed"` +} + +func main() { + baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + clients := flag.Int("clients", 10, "number of concurrent clients") + perClient := flag.Int("per-client", 10, "messages sent by each client") + output := flag.String("output", "", "optional path for the JSON report") + flag.Parse() + + report, err := run(*baseURL, *clients, *perClient) + if err != nil { + fail(err) + } + text, _ := json.MarshalIndent(report, "", " ") + fmt.Println(string(text)) + if *output != "" { + if err := writeReport(*output, text); err != nil { + fail(err) + } + } + if !report.Passed { + fail(fmt.Errorf("concurrency invariants violated")) + } + fmt.Printf("PASS: %d clients x %d messages produced exactly %d unique ordered messages\n", + report.Clients, report.PerClient, report.Unique) +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "FAIL:", err) + os.Exit(1) +} + +func writeReport(path string, text []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, append(text, '\n'), 0o644) +} + +func run(baseURL string, clients, perClient int) (Report, error) { + if clients < 1 || perClient < 1 { + return Report{}, fmt.Errorf("clients and per-client must be positive") + } + report := Report{ + BaseURL: baseURL, + StartedAt: time.Now().UTC(), + Clients: clients, + PerClient: perClient, + Expected: clients * perClient, + } + api := newClient(baseURL) + runID := uuid.NewString()[:8] + + members, tokens, err := registerClients(api, runID, clients) + if err != nil { + return report, err + } + room, err := createChat(api, runID, members, tokens[0]) + if err != nil { + return report, err + } + + started := time.Now() + created, failed := sendConcurrently(baseURL, room.ID, tokens, perClient) + report.ElapsedSeconds = time.Since(started).Seconds() + report.Created = len(created) + report.Failed = failed + + listed, pages, err := readHistory(api, room.ID, tokens[0]) + if err != nil { + return report, err + } + report.Pages = pages + report.Listed = len(listed) + report.Unique, report.Missing = compare(created, listed) + report.Ordered = sort.StringsAreSorted(listed) + report.Passed = report.Failed == 0 && + report.Created == report.Expected && + report.Listed == report.Expected && + report.Unique == report.Expected && + report.Missing == 0 && + report.Ordered + return report, nil +} + +// clientPassword is shared by the synthetic clients of one run. +const clientPassword = "concurrency-check-password" + +// registerClients registers and logs in n clients, returning their ids and +// bearer tokens in the same order. +func registerClients(api *client, runID string, n int) (ids, tokens []string, err error) { + for i := range n { + var created user.Response + username := fmt.Sprintf("load_%s_%d", runID, i) + body := map[string]string{ + "username": username, + "display_name": fmt.Sprintf("Load client %d", i), + "password": clientPassword, + } + if err := api.post("/api/v1/users", "", body, &created); err != nil { + return nil, nil, fmt.Errorf("register client %d: %w", i, err) + } + token, err := api.login(username, clientPassword) + if err != nil { + return nil, nil, fmt.Errorf("log in client %d: %w", i, err) + } + ids = append(ids, created.ID) + tokens = append(tokens, token) + } + return ids, tokens, nil +} + +func createChat(api *client, runID string, members []string, creatorToken string) (chat.ChatResponse, error) { + var created chat.ChatResponse + body := map[string]any{"title": "Concurrency check " + runID, "member_ids": members} + if err := api.post("/api/v1/chats", creatorToken, body, &created); err != nil { + return chat.ChatResponse{}, fmt.Errorf("create chat: %w", err) + } + return created, nil +} + +// sendConcurrently gives every member its own HTTP client and releases them +// together, so the server sees genuinely parallel writes to one chat. +func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (map[string]struct{}, int) { + var mu sync.Mutex + created := make(map[string]struct{}, len(tokens)*perClient) + failed := 0 + + var wg sync.WaitGroup + start := make(chan struct{}) + for i, token := range tokens { + wg.Add(1) + go func(n int, token string) { + defer wg.Done() + own := newClient(baseURL) + <-start + for m := range perClient { + var msg chat.MessageResponse + body := map[string]string{"text": fmt.Sprintf("client %d message %d", n, m)} + err := own.post("/api/v1/chats/"+chatID+"/messages", token, body, &msg) + mu.Lock() + if err != nil { + failed++ + } else { + created[msg.ID] = struct{}{} + } + mu.Unlock() + } + }(i, token) + } + close(start) + wg.Wait() + return created, failed +} + +func readHistory(api *client, chatID, token string) ([]string, int, error) { + var ids []string + pages := 0 + cursor := "" + for { + var page pagination.PageResponse[chat.MessageResponse] + if err := api.get("/api/v1/chats/"+chatID+"/messages", token, cursor, &page); err != nil { + return nil, pages, fmt.Errorf("read history: %w", err) + } + pages++ + for _, m := range page.Items { + ids = append(ids, m.ID) + } + if page.NextCursor == nil { + return ids, pages, nil + } + cursor = *page.NextCursor + } +} + +// compare returns how many distinct ids the history holds and how many +// acknowledged ids never showed up in it. +func compare(created map[string]struct{}, listed []string) (unique, missing int) { + seen := make(map[string]struct{}, len(listed)) + for _, id := range listed { + seen[id] = struct{}{} + } + for id := range created { + if _, ok := seen[id]; !ok { + missing++ + } + } + return len(seen), missing +} diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go new file mode 100644 index 00000000..eb607c3d --- /dev/null +++ b/tools/concurrency/main_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "os" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func TestRunAgainstLiveServer(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 10, 10) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Clients != 10 || report.PerClient != 10 || report.Expected != 100 { + t.Fatalf("report = %+v", report) + } + if report.Created != 100 || report.Listed != 100 || report.Unique != 100 || report.Missing != 0 { + t.Fatalf("report = %+v", report) + } + if !report.Ordered || !report.Passed { + t.Fatalf("report = %+v", report) + } +} + +func TestRunFailsOnUnreachableServer(t *testing.T) { + if _, err := run("http://127.0.0.1:1", 2, 2); err == nil { + t.Fatal("expected an error for an unreachable server") + } +} -- 2.54.0 From 7fa8881eb329b5c43d0601a3abe05f8564589c3b Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 16:58:11 +0300 Subject: [PATCH 02/21] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20X-Instance-Id,=20=D0=B7=D0=B0=D0=B4=D0=B5=D1=80?= =?UTF-8?q?=D0=B6=D0=BA=D1=83=20drain=20=D0=BF=D1=80=D0=B8=20SIGTERM=20?= =?UTF-8?q?=D0=B8=20=D0=BE=D1=82=D0=B4=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9?= =?UTF-8?q?=20=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA=20=D0=BC=D0=B8=D0=B3?= =?UTF-8?q?=D1=80=D0=B0=D1=86=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 13 ++-- cmd/migrate/main.go | 58 ++++++++++++++ cmd/server/main.go | 39 +++++----- compose.yaml | 17 +++++ internal/app/app.go | 11 ++- internal/app/app_test.go | 7 ++ internal/app/auth_test.go | 7 +- internal/app/failure_test.go | 1 + internal/platform/config/config.go | 83 ++++++++++++++++++--- internal/platform/config/config_test.go | 58 ++++++++++++-- internal/platform/httpx/engine.go | 7 +- internal/platform/httpx/httpx_test.go | 23 +++++- internal/platform/httpx/middleware.go | 9 +++ internal/platform/ident/ident.go | 11 +++ internal/platform/ident/ident_test.go | 12 +++ internal/platform/postgres/postgres.go | 10 +-- internal/platform/postgres/postgres_test.go | 15 +++- internal/server/server.go | 19 +++-- internal/server/server_test.go | 49 +++++++++++- internal/testenv/testenv.go | 2 +- tools/concurrency/main_test.go | 2 +- 21 files changed, 389 insertions(+), 64 deletions(-) create mode 100644 cmd/migrate/main.go diff --git a/Dockerfile b/Dockerfile index b8bc70b1..571f6bb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,23 @@ -# Build stage: compile a static binary with the exact toolchain pinned in go.mod. +# Build stage: compile static binaries with the exact toolchain pinned in go.mod. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download && go mod verify COPY . . -RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server +RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Runtime stage: minimal image, unprivileged user, the server is PID 1. +# Runtime stage: minimal image, unprivileged user, the server is PID 1. The +# same image runs the one-shot migrate service with its own entrypoint. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app -COPY --from=build /out/server /usr/local/bin/server +COPY --from=build /out/server /out/migrate /usr/local/bin/ USER app ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab EXPOSE 8080 -HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 \ +# The short start interval lets `compose up --wait` and the rolling update see +# a new replica as healthy about a second after it is ready. +HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 ENTRYPOINT ["/usr/local/bin/server"] diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..201cec5d --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,58 @@ +// Command migrate applies the embedded schema migrations once and exits. It +// runs as a one-shot service before the API replicas start, so the replicas +// never race each other over the schema and need no rights to change it. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" +) + +// poolSize covers the goose session lock and the migration itself. +const poolSize = 2 + +func main() { + cfg, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + log := newLogger(cfg) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg); err != nil { + log.Error("migration failed", "error", err.Error()) + os.Exit(1) + } +} + +func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, poolSize) + if err != nil { + return err + } + defer pool.Close() + applied, err := postgres.Migrate(ctx, pool) + if err != nil { + return err + } + log.Info("migrations applied", "applied_migrations", applied) + return nil +} + +func newLogger(cfg config.Migrate) *slog.Logger { + opts := &slog.HandlerOptions{Level: cfg.LogLevel} + if cfg.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 305e4ff2..6964a810 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -8,10 +8,12 @@ import ( "net" "os" "os/signal" + "runtime" "syscall" "messenger/internal/app" "messenger/internal/platform/config" + "messenger/internal/platform/ident" "messenger/internal/platform/postgres" "messenger/internal/platform/valkey" "messenger/internal/server" @@ -23,30 +25,29 @@ func main() { fmt.Fprintln(os.Stderr, "configuration error:", err) os.Exit(2) } - log := newLogger(cfg) + // Every log line carries the alias that this process sends as + // X-Instance-Id, so a response can be matched to its replica's log. + alias := ident.NewAlias() + log := newLogger(cfg).With("instance", alias) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() - if err := run(ctx, log, cfg); err != nil { + if err := run(ctx, log, cfg, alias); err != nil { log.Error("server stopped with error", "error", err.Error()) os.Exit(1) } } // run connects the dependencies before it listens, so the port accepts -// connections only once the schema is in place. -func run(ctx context.Context, log *slog.Logger, cfg config.Config) error { - pool, err := postgres.NewPool(ctx, cfg.DatabaseURL) +// connections only once they answer. The schema is applied beforehand by the +// separate migrate command, once per deployment rather than by every replica. +func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns) if err != nil { return err } defer pool.Close() - applied, err := postgres.Migrate(ctx, pool) - if err != nil { - return err - } - log.Info("schema ready", "applied_migrations", applied) sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL) if err != nil { return err @@ -58,15 +59,19 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config) error { return fmt.Errorf("listen on %s: %w", cfg.Addr(), err) } log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(), - "shutdown_timeout", cfg.ShutdownTimeout.String(), "session_ttl", cfg.SessionTTL.String()) + "gomaxprocs", runtime.GOMAXPROCS(0), "db_max_conns", cfg.DBMaxConns, + "drain_delay", cfg.DrainDelay.String(), "shutdown_timeout", cfg.ShutdownTimeout.String(), + "session_ttl", cfg.SessionTTL.String(), "idempotency_ttl", cfg.IdempotencyTTL.String()) a := app.New(app.Deps{ - Log: log, - DB: pool, - Sessions: sessions, - SessionTTL: cfg.SessionTTL, - MaxBodyBytes: cfg.MaxBodyBytes, + Log: log, + DB: pool, + Sessions: sessions, + SessionTTL: cfg.SessionTTL, + MaxBodyBytes: cfg.MaxBodyBytes, + InstanceID: alias, + IdempotencyTTL: cfg.IdempotencyTTL, }) - return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.ShutdownTimeout) + return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) } func newLogger(cfg config.Config) *slog.Logger { diff --git a/compose.yaml b/compose.yaml index 0a11d1f6..9228acc9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,8 @@ services: - database_url - session_store_url depends_on: + migrate: + condition: service_completed_successfully postgres: condition: service_healthy valkey: @@ -28,6 +30,21 @@ services: - "${PUBLISH:-127.0.0.1:8080}:8080" stop_grace_period: 10s + # One-shot: applies the schema before the API starts, then exits. + migrate: + image: mtusi-messenger:lab3 + entrypoint: ["/usr/local/bin/migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 environment: diff --git a/internal/app/app.go b/internal/app/app.go index 6653c8c5..288122a2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -25,6 +25,8 @@ import ( type App struct { Engine *gin.Engine Health *health.Handler + // Chats is exposed for the background cleanup of idempotency keys. + Chats *chat.Service } // Deps are the shared resources the modules are built on; their lifetime is @@ -37,6 +39,11 @@ type Deps struct { MaxBodyBytes int64 // RequestTimeout bounds every API request; zero means the default. RequestTimeout time.Duration + // InstanceID is sent as X-Instance-Id on every response. + InstanceID string + // IdempotencyTTL is how long a message Idempotency-Key is remembered; + // zero means the default. + IdempotencyTTL time.Duration } // defaultRequestTimeout stays below the server's 10s write timeout, so a @@ -47,7 +54,7 @@ func New(d Deps) *App { users := user.NewService(userstore.NewUsers(d.DB)) chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users) - engine := httpx.NewEngine(d.Log, d.MaxBodyBytes) + engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) probes := health.New( health.Check{Name: "postgres", Probe: d.DB.Ping}, health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, @@ -66,5 +73,5 @@ func New(d Deps) *App { user.NewHandler(users).Register(public, protected) chat.NewHandler(chats).Register(protected) - return &App{Engine: engine, Health: probes} + return &App{Engine: engine, Health: probes, Chats: chats} } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index b9ba72b6..e908cf83 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -50,6 +50,9 @@ func TestMain(m *testing.M) { const testPassword = "correct-horse-battery" +// testInstance is the alias of the in-process application under test. +const testInstance = "00000000000000a1" + // errorBody mirrors the contract error envelope. type errorBody struct { Error struct { @@ -103,6 +106,7 @@ func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester { Sessions: kv.Client, SessionTTL: ttl, MaxBodyBytes: 64 * 1024, + InstanceID: testInstance, })} } @@ -135,6 +139,9 @@ func (ts *tester) do(method, path, token string, body any, headers map[string]st if res.requestID == "" { ts.t.Fatalf("%s %s: missing X-Request-Id", method, path) } + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + ts.t.Fatalf("%s %s: X-Instance-Id = %q, want %q", method, path, got, testInstance) + } if res.status >= 400 { var eb errorBody if err := json.Unmarshal(res.raw, &eb); err != nil { diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go index 602d1e2a..727cd6a3 100644 --- a/internal/app/auth_test.go +++ b/internal/app/auth_test.go @@ -146,11 +146,14 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") } +// The TTL leaves room for a loaded machine: with one second the first request +// after login sometimes arrived too late under -race with all packages +// running at once. func TestSessionExpiresAfterTTL(t *testing.T) { - ts := newTesterWithTTL(t, time.Second) + ts := newTesterWithTTL(t, 3*time.Second) alice := ts.user("alice", "A") ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") - time.Sleep(1200 * time.Millisecond) + time.Sleep(3200 * time.Millisecond) ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized") } diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go index cadc4ee6..155c8000 100644 --- a/internal/app/failure_test.go +++ b/internal/app/failure_test.go @@ -35,6 +35,7 @@ func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *te SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, RequestTimeout: requestTimeout, + InstanceID: testInstance, }) return &tester{t: t, a: a}, pg, vk } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 30100b71..0dcc0d9d 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -25,6 +25,22 @@ type Config struct { DatabaseURL string SessionStoreURL string SessionTTL time.Duration + // DrainDelay is how long a stopping process keeps serving with readiness + // already off, so the load balancer can take it out of rotation before + // the listener closes. It must exceed the balancer's detection time. + DrainDelay time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + // DebugAddr enables the internal expvar listener; empty disables it. + DebugAddr string +} + +// Migrate is the configuration of the one-shot migration command, which +// needs the database only. +type Migrate struct { + Env string + LogLevel slog.Level + DatabaseURL string } // Lookup reads one variable, reporting whether it was set; os.LookupEnv @@ -40,6 +56,9 @@ func Load(lookup Lookup) (Config, error) { ShutdownTimeout: 8 * time.Second, MaxBodyBytes: 64 * 1024, SessionTTL: time.Hour, + DrainDelay: 2 * time.Second, + IdempotencyTTL: 24 * time.Hour, + DBMaxConns: 10, } if v, ok := lookup("APP_HOST"); ok { if strings.TrimSpace(v) == "" { @@ -54,15 +73,9 @@ func Load(lookup Lookup) (Config, error) { } cfg.Port = port } - if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { - cfg.Env = v - } - if v, ok := lookup("APP_LOG_LEVEL"); ok { - var level slog.Level - if err := level.UnmarshalText([]byte(v)); err != nil { - return Config{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) - } - cfg.LogLevel = level + var err error + if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + return Config{}, err } if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { d, err := time.ParseDuration(v) @@ -85,7 +98,33 @@ func Load(lookup Lookup) (Config, error) { } cfg.SessionTTL = time.Duration(n) * time.Second } - var err error + if v, ok := lookup("APP_DRAIN_DELAY"); ok { + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return Config{}, fmt.Errorf("APP_DRAIN_DELAY must be a non-negative duration such as 2s, got %q", v) + } + cfg.DrainDelay = d + } + if v, ok := lookup("IDEMPOTENCY_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.IdempotencyTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("DB_MAX_CONNS"); ok { + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 || n > 1000 { + return Config{}, fmt.Errorf("DB_MAX_CONNS must be an integer between 1 and 1000, got %q", v) + } + cfg.DBMaxConns = int32(n) + } + if v, ok := lookup("APP_DEBUG_ADDR"); ok && v != "" { + if _, _, err := net.SplitHostPort(v); err != nil { + return Config{}, fmt.Errorf("APP_DEBUG_ADDR must be host:port such as :9090, got %q", v) + } + cfg.DebugAddr = v + } if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { return Config{}, err } @@ -95,6 +134,30 @@ func Load(lookup Lookup) (Config, error) { return cfg, nil } +func LoadMigrate(lookup Lookup) (Migrate, error) { + cfg := Migrate{Env: "lab", LogLevel: slog.LevelInfo} + var err error + if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + return Migrate{}, err + } + if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { + return Migrate{}, err + } + return cfg, nil +} + +func logging(lookup Lookup, env string, level slog.Level) (string, slog.Level, error) { + if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { + env = v + } + if v, ok := lookup("APP_LOG_LEVEL"); ok { + if err := level.UnmarshalText([]byte(v)); err != nil { + return "", 0, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + } + } + return env, level, nil +} + // secret reads the file named by the variable; surrounding whitespace, // including the trailing newline editors add, is dropped. func secret(lookup Lookup, key string) (string, error) { diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go index 2288dcd2..4900ff3d 100644 --- a/internal/platform/config/config_test.go +++ b/internal/platform/config/config_test.go @@ -65,6 +65,9 @@ func TestDefaults(t *testing.T) { if cfg.SessionTTL != time.Hour { t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL) } + if cfg.DrainDelay != 2*time.Second || cfg.IdempotencyTTL != 24*time.Hour || cfg.DBMaxConns != 10 || cfg.DebugAddr != "" { + t.Fatalf("lab3 defaults = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } } func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { @@ -92,13 +95,17 @@ func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { func TestOverrides(t *testing.T) { cfg, err := config.Load(env(required(t, map[string]string{ - "APP_HOST": "127.0.0.1", - "APP_PORT": "9090", - "APP_ENV": "dev", - "APP_LOG_LEVEL": "debug", - "APP_SHUTDOWN_TIMEOUT": "3s", - "APP_MAX_BODY_BYTES": "1024", - "SESSION_TTL_SECONDS": "300", + "APP_HOST": "127.0.0.1", + "APP_PORT": "9090", + "APP_ENV": "dev", + "APP_LOG_LEVEL": "debug", + "APP_SHUTDOWN_TIMEOUT": "3s", + "APP_MAX_BODY_BYTES": "1024", + "SESSION_TTL_SECONDS": "300", + "APP_DRAIN_DELAY": "0s", + "IDEMPOTENCY_TTL_SECONDS": "60", + "DB_MAX_CONNS": "25", + "APP_DEBUG_ADDR": ":9090", }))) if err != nil { t.Fatalf("Load error: %v", err) @@ -109,6 +116,9 @@ func TestOverrides(t *testing.T) { if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { t.Fatalf("cfg = %+v", cfg) } + if cfg.DrainDelay != 0 || cfg.IdempotencyTTL != time.Minute || cfg.DBMaxConns != 25 || cfg.DebugAddr != ":9090" { + t.Fatalf("lab3 overrides = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } } func TestInvalidValuesAreReported(t *testing.T) { @@ -126,10 +136,42 @@ func TestInvalidValuesAreReported(t *testing.T) { "ttl negative": {"SESSION_TTL_SECONDS": "-5"}, "ttl text": {"SESSION_TTL_SECONDS": "hour"}, "ttl fraction": {"SESSION_TTL_SECONDS": "1.5"}, + "drain negative": {"APP_DRAIN_DELAY": "-1s"}, + "drain text": {"APP_DRAIN_DELAY": "abc"}, + "key ttl zero": {"IDEMPOTENCY_TTL_SECONDS": "0"}, + "key ttl text": {"IDEMPOTENCY_TTL_SECONDS": "day"}, + "pool zero": {"DB_MAX_CONNS": "0"}, + "pool too big": {"DB_MAX_CONNS": "1001"}, + "pool text": {"DB_MAX_CONNS": "many"}, + "debug addr": {"APP_DEBUG_ADDR": "nonsense"}, } for name, values := range cases { - if _, err := config.Load(env(required(t, values))); err == nil { + _, err := config.Load(env(required(t, values))) + if err == nil { t.Errorf("%s: want error", name) + continue + } + for key := range values { + if !strings.Contains(err.Error(), key) { + t.Errorf("%s: error %q does not name %s", name, err, key) + } } } } + +func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) { + cfg, err := config.LoadMigrate(env(map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "APP_ENV": "dev", + "APP_LOG_LEVEL": "warn", + })) + if err != nil { + t.Fatalf("LoadMigrate error: %v", err) + } + if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.LogLevel != slog.LevelWarn { + t.Fatalf("cfg = %+v", cfg) + } + if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") { + t.Fatalf("missing database url: err = %v", err) + } +} diff --git a/internal/platform/httpx/engine.go b/internal/platform/httpx/engine.go index ba56a897..4f1f89ad 100644 --- a/internal/platform/httpx/engine.go +++ b/internal/platform/httpx/engine.go @@ -20,8 +20,9 @@ const ( ) // NewEngine builds an engine whose every response, including routing -// failures and panics, carries X-Request-Id and the contract error body. -func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine { +// failures and panics, carries X-Instance-Id, X-Request-Id and the contract +// error body. +func NewEngine(log *slog.Logger, maxBodyBytes int64, instanceID string) *gin.Engine { gin.SetMode(gin.ReleaseMode) e := gin.New() // Redirects would answer with text/plain and without a request id. @@ -29,7 +30,7 @@ func NewEngine(log *slog.Logger, maxBodyBytes int64) *gin.Engine { e.RedirectFixedPath = false e.HandleMethodNotAllowed = true - e.Use(requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) + e.Use(instance(instanceID), requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) }) e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") }) return e diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go index a745d5a9..66244ad7 100644 --- a/internal/platform/httpx/httpx_test.go +++ b/internal/platform/httpx/httpx_test.go @@ -33,6 +33,8 @@ type payload struct { Count int `json:"count"` } +const testInstance = "0123456789abcdef" + func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine { t.Helper() return newEngineAt(t, logs, slog.LevelDebug) @@ -44,7 +46,7 @@ func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine logs = &bytes.Buffer{} } log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level})) - e := httpx.NewEngine(log, 64) + e := httpx.NewEngine(log, 64, testInstance) e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": ""}) }) e.GET("/panic", func(c *gin.Context) { panic("secret detail") }) e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) }) @@ -106,6 +108,25 @@ func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, sta } } +// The balancer spreads one client's requests over the replicas; the header +// is how a client tells which process answered, so no response may lack it. +func TestEveryResponseCarriesInstanceID(t *testing.T) { + e := newEngine(t, nil) + for _, r := range []struct{ method, path string }{ + {http.MethodGet, "/ok"}, + {http.MethodGet, "/missing"}, + {http.MethodDelete, "/ok"}, + {http.MethodGet, "/panic"}, + {http.MethodGet, "/fail"}, + {http.MethodPost, "/decode"}, + } { + rec, _ := do(t, e, r.method, r.path, "", nil) + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + t.Errorf("%s %s (status %d): X-Instance-Id = %q, want %q", r.method, r.path, rec.Code, got, testInstance) + } + } +} + func TestUnknownRouteIsJSON404(t *testing.T) { rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil) assertError(t, rec, eb, http.StatusNotFound, "not_found") diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index d153a980..a0a9cc6d 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -13,6 +13,15 @@ import ( "github.com/gin-gonic/gin" ) +// instance names the process that answered, so a client behind the load +// balancer can tell the replicas apart. +func instance(id string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("X-Instance-Id", id) + c.Next() + } +} + // requestID assigns a fresh identifier to every request. // // An incoming X-Request-Id is deliberately not reused. This service is diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go index 121500d9..b70efbe9 100644 --- a/internal/platform/ident/ident.go +++ b/internal/platform/ident/ident.go @@ -7,6 +7,8 @@ package ident import ( + "crypto/rand" + "encoding/hex" "errors" "time" @@ -25,6 +27,15 @@ func New() uuid.UUID { return id } +// NewAlias names one server process for X-Instance-Id. It is random rather +// than the hostname, so the header reveals nothing about the infrastructure, +// and a restarted process gets a new one. +func NewAlias() string { + var raw [8]byte + _, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24. + return hex.EncodeToString(raw[:]) +} + // TimeOf extracts the creation time carried by a UUIDv7 as UTC with // millisecond precision. func TimeOf(id uuid.UUID) time.Time { diff --git a/internal/platform/ident/ident_test.go b/internal/platform/ident/ident_test.go index 1ae020df..d0a57d4d 100644 --- a/internal/platform/ident/ident_test.go +++ b/internal/platform/ident/ident_test.go @@ -2,6 +2,7 @@ package ident_test import ( "bytes" + "regexp" "testing" "time" @@ -70,3 +71,14 @@ func TestParseRejectsNonCanonicalForms(t *testing.T) { } } } + +func TestNewAliasIs16HexAndRandom(t *testing.T) { + a, b := ident.NewAlias(), ident.NewAlias() + hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`) + if !hex16.MatchString(a) || !hex16.MatchString(b) { + t.Fatalf("aliases %q, %q are not 16 lowercase hex characters", a, b) + } + if a == b { + t.Fatalf("two aliases are equal: %q", a) + } +} diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go index 616caf62..6409553e 100644 --- a/internal/platform/postgres/postgres.go +++ b/internal/platform/postgres/postgres.go @@ -1,8 +1,9 @@ // Package postgres opens the connection pool and applies the schema. // -// Pool size and server-side timeouts are constants rather than settings: they -// bound how long one request can hold a connection, and the request deadline -// of the API is chosen against them. +// Server-side timeouts are constants rather than settings: they bound how +// long one request can hold a connection, and the request deadline of the API +// is chosen against them. The pool size is a setting, because the scaling +// study varies it. package postgres import ( @@ -27,7 +28,6 @@ import ( ) const ( - maxConns = 10 connectTimeout = time.Second statementTimeout = 5 * time.Second idleInTxTimeout = 10 * time.Second @@ -35,7 +35,7 @@ const ( const codeUniqueViolation = "23505" -func NewPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) { +func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, error) { cfg, err := pgxpool.ParseConfig(dsn) if err != nil { return nil, fmt.Errorf("parse database url: %w", err) diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go index c3a14873..49dc627a 100644 --- a/internal/platform/postgres/postgres_test.go +++ b/internal/platform/postgres/postgres_test.go @@ -184,8 +184,21 @@ func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { } } +// The pool size is an experiment variable of the scaling study, so it is +// passed in rather than fixed. +func TestNewPoolUsesTheRequestedSize(t *testing.T) { + pool, err := postgres.NewPool(context.Background(), db.DSN, 3) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if got := pool.Config().MaxConns; got != 3 { + t.Fatalf("MaxConns = %d, want 3", got) + } +} + func TestClassifyRealConnectFailure(t *testing.T) { - _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1") + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1", 1) if err == nil { t.Fatal("NewPool to a closed port must fail") } diff --git a/internal/server/server.go b/internal/server/server.go index 8a70e0fe..718c66df 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,6 +1,7 @@ // Package server runs the HTTP listener with the lifecycle the lab -// requires: readiness drops first, the listener stops accepting, in-flight -// requests drain within a bounded timeout, then the process exits. +// requires: readiness drops first, the process keeps serving for a drain +// delay while the load balancer notices, then the listener stops accepting, +// in-flight requests drain within a bounded timeout and the process exits. package server import ( @@ -17,8 +18,15 @@ type Readiness interface { SetReady(ready bool) } -// Run serves until ctx is cancelled, then shuts down within shutdownTimeout. -func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, shutdownTimeout time.Duration) error { +// Run serves until ctx is cancelled, keeps serving for drainDelay with +// readiness off, then shuts down within shutdownTimeout. +// +// The balancer learns about a stopping replica only from its failing +// readiness checks. Closing the listener at once would let it send requests +// to a closed port, or over a kept-alive connection the server is closing, +// until the checks catch up. drainDelay must therefore exceed the balancer's +// detection time. +func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error { srv := &http.Server{ Handler: h, ReadHeaderTimeout: 5 * time.Second, @@ -45,8 +53,9 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, case <-ctx.Done(): } - log.Info("shutdown started", "timeout", shutdownTimeout.String()) + log.Info("shutdown started", "drain_delay", drainDelay.String(), "timeout", shutdownTimeout.String()) ready.SetReady(false) + time.Sleep(drainDelay) drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() if err := srv.Shutdown(drain); err != nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 040f172a..4f447f85 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -19,7 +19,7 @@ import ( // newHandler builds the smallest handler with probes; the lifecycle does not // depend on the business modules or their stores. func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { - e := httpx.NewEngine(log, 1024) + e := httpx.NewEngine(log, 1024, "test") probes := health.New() probes.Register(e) return e, probes @@ -34,7 +34,7 @@ func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() base := "http://" + ln.Addr().String() waitFor(t, base+"/health/ready", http.StatusOK) @@ -62,7 +62,7 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- server.Run(ctx, log, ln, h, probes, 5*time.Second) }() + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() base := "http://" + ln.Addr().String() waitFor(t, base+"/health/ready", http.StatusOK) @@ -85,6 +85,49 @@ func TestReadinessDropsBeforeListenerCloses(t *testing.T) { t.Fatal("readiness must be switched off during shutdown") } +// During the drain delay the balancer is still learning that the replica is +// going away: readiness already fails, yet new connections are accepted and +// served. Only after the delay does the listener close. +func TestDrainKeepsServingWhileNotReady(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + const drain = 700 * time.Millisecond + go func() { done <- server.Run(ctx, log, ln, h, probes, drain, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + waitFor(t, base+"/health/ready", http.StatusServiceUnavailable) + // A fresh connection, as the balancer opens for a new request. + fresh := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + resp, err := fresh.Get(base + "/health/live") + if err != nil { + t.Fatalf("new connection during the drain delay: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("liveness during drain = %d, want 200", resp.StatusCode) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop after the drain delay") + } + if _, err := fresh.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after the drain delay") + } +} + func waitFor(t *testing.T, url string, status int) { t.Helper() deadline := time.Now().Add(5 * time.Second) diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index 980d191d..57cf53bd 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -62,7 +62,7 @@ func StartPostgres(ctx context.Context) (*DB, error) { db.Close() return nil, fmt.Errorf("postgres address: %w", err) } - db.Pool, err = postgres.NewPool(ctx, db.DSN) + db.Pool, err = postgres.NewPool(ctx, db.DSN, 10) if err != nil { db.Close() return nil, err diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go index eb607c3d..7c9158b4 100644 --- a/tools/concurrency/main_test.go +++ b/tools/concurrency/main_test.go @@ -39,7 +39,7 @@ func TestMain(m *testing.M) { func TestRunAgainstLiveServer(t *testing.T) { log := slog.New(slog.NewTextHandler(io.Discard, nil)) - a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024}) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) srv := httptest.NewServer(a.Engine) defer srv.Close() -- 2.54.0 From 951fd0d517e7c5cec8fc6f1c80814f1ca38e7b90 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 17:11:04 +0300 Subject: [PATCH 03/21] =?UTF-8?q?=D0=A1=D0=B4=D0=B5=D0=BB=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BE=D1=82=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D1=83=20?= =?UTF-8?q?=D1=81=D0=BE=D0=BE=D0=B1=D1=89=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B8?= =?UTF-8?q?=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=20=D0=BF=D0=BE=20Idempotency-Key=20=D1=81=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=89=D0=B5=D0=B9=20=D1=82=D0=B0=D0=B1=D0=BB=D0=B8=D1=86?= =?UTF-8?q?=D0=B5=D0=B9=20=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=B9=20=D0=B8=20?= =?UTF-8?q?=D0=B8=D1=85=20=D1=87=D0=B8=D1=81=D1=82=D0=BA=D0=BE=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/server/main.go | 10 + internal/app/app.go | 9 +- internal/app/app_test.go | 13 +- internal/app/auth_test.go | 26 +- internal/app/idempotency_test.go | 154 +++++++++++ internal/chat/chat.go | 4 + internal/chat/cleanup.go | 33 +++ internal/chat/handler.go | 22 +- internal/chat/repository.go | 9 +- internal/chat/service.go | 42 ++- internal/chat/service_test.go | 148 +++++++++- internal/chat/store/export_test.go | 9 + internal/chat/store/idempotency_test.go | 252 ++++++++++++++++++ internal/chat/store/postgres.go | 91 ++++++- internal/chat/store/postgres_test.go | 17 +- internal/platform/postgres/postgres_test.go | 12 +- migrations/00002_message_idempotency_keys.sql | 25 ++ tools/concurrency/client.go | 17 ++ tools/concurrency/main.go | 3 +- 19 files changed, 856 insertions(+), 40 deletions(-) create mode 100644 internal/app/idempotency_test.go create mode 100644 internal/chat/cleanup.go create mode 100644 internal/chat/store/export_test.go create mode 100644 internal/chat/store/idempotency_test.go create mode 100644 migrations/00002_message_idempotency_keys.sql diff --git a/cmd/server/main.go b/cmd/server/main.go index 6964a810..f3cbafcf 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -9,9 +9,12 @@ import ( "os" "os/signal" "runtime" + "sync" "syscall" + "time" "messenger/internal/app" + "messenger/internal/chat" "messenger/internal/platform/config" "messenger/internal/platform/ident" "messenger/internal/platform/postgres" @@ -19,6 +22,9 @@ import ( "messenger/internal/server" ) +// keyCleanupEvery is how often each replica deletes expired Idempotency-Keys. +const keyCleanupEvery = 10 * time.Minute + func main() { cfg, err := config.Load(os.LookupEnv) if err != nil { @@ -71,6 +77,10 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, }) + var background sync.WaitGroup + defer background.Wait() // before the deferred pool.Close + background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) } diff --git a/internal/app/app.go b/internal/app/app.go index 288122a2..24dd4c16 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -50,9 +50,16 @@ type Deps struct { // request that gives up on a dependency can still write its 503. const defaultRequestTimeout = 5 * time.Second +// defaultIdempotencyTTL is the contract's retention of message keys. +const defaultIdempotencyTTL = 24 * time.Hour + func New(d Deps) *App { users := user.NewService(userstore.NewUsers(d.DB)) - chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users) + keyTTL := d.IdempotencyTTL + if keyTTL == 0 { + keyTTL = defaultIdempotencyTTL + } + chats := chat.NewService(chatstore.NewChats(d.DB), chatstore.NewMessages(d.DB), users, keyTTL) engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) probes := health.New( diff --git a/internal/app/app_test.go b/internal/app/app_test.go index e908cf83..622d46c6 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -18,6 +18,8 @@ import ( "testing" "time" + "github.com/google/uuid" + "messenger/internal/app" "messenger/internal/chat" "messenger/internal/platform/pagination" @@ -238,11 +240,16 @@ func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse func (ts *tester) message(token, chatID, text string) chat.MessageResponse { ts.t.Helper() - res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, nil) + res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, freshKey()) ts.expect(res, http.StatusCreated, "") return decode[chat.MessageResponse](ts.t, res) } +// freshKey is the header of a first attempt: a key never used before. +func freshKey() map[string]string { + return map[string]string{"Idempotency-Key": uuid.NewString()} +} + func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] { ts.t.Helper() res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil) @@ -418,7 +425,7 @@ func TestMessagesEndToEnd(t *testing.T) { t.Fatalf("page2 = %+v", page2) } - post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, nil) } + post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, freshKey()) } ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request") ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request") ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request") @@ -480,7 +487,7 @@ func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) { for n, c := range clientsList { wg.Go(func() { for i := 0; i < perClient; i++ { - res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, nil) + res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, freshKey()) if res.status != http.StatusCreated { t.Errorf("client %d: status %d body %s", n, res.status, res.raw) } diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go index 727cd6a3..2ed17d62 100644 --- a/internal/app/auth_test.go +++ b/internal/app/auth_test.go @@ -146,15 +146,29 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") } -// The TTL leaves room for a loaded machine: with one second the first request -// after login sometimes arrived too late under -race with all packages -// running at once. +// The expiry is an absolute time written by the application and enforced by +// Valkey's clock. Here the test runs on the Windows host and Valkey in the +// Docker VM, whose clock lags by a fraction of a second, so the test waits +// for the 401 with a tolerance instead of sleeping exactly one TTL. On the +// stand both run in the same VM. func TestSessionExpiresAfterTTL(t *testing.T) { - ts := newTesterWithTTL(t, 3*time.Second) + const ttl, clockTolerance = 2 * time.Second, 3 * time.Second + ts := newTesterWithTTL(t, ttl) alice := ts.user("alice", "A") ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") - time.Sleep(3200 * time.Millisecond) - ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusUnauthorized, "unauthorized") + time.Sleep(ttl) + deadline := time.Now().Add(clockTolerance) + for { + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + if res.status == http.StatusUnauthorized { + ts.expect(res, http.StatusUnauthorized, "unauthorized") + return + } + if time.Now().After(deadline) { + t.Fatalf("session still valid %v after its TTL of %v", clockTolerance, ttl) + } + time.Sleep(100 * time.Millisecond) + } } func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) { diff --git a/internal/app/idempotency_test.go b/internal/app/idempotency_test.go new file mode 100644 index 00000000..9da80609 --- /dev/null +++ b/internal/app/idempotency_test.go @@ -0,0 +1,154 @@ +package app_test + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "messenger/internal/app" +) + +func (ts *tester) send(token, chatID, key string, body any) response { + ts.t.Helper() + return ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, body, map[string]string{"Idempotency-Key": key}) +} + +func TestSendIdempotencyKeyValidation(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "ключи", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + body := map[string]string{"text": "x"} + + ts.expect(ts.do(http.MethodPost, path, alice.token, body, nil), http.StatusBadRequest, "invalid_request") + for name, key := range map[string]string{ + "empty": "", + "too long": strings.Repeat("a", 129), + "inner space": "a b", + "non-ascii": "ключ", + "slash": "a/b", + } { + res := ts.send(alice.token, room.ID, key, body) + if res.status != http.StatusBadRequest { + t.Errorf("%s key: status %d, want 400", name, res.status) + } + } + ts.expect(ts.send(alice.token, room.ID, strings.Repeat("a", 128), body), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "Az09._:-", body), http.StatusCreated, "") + + // Two Idempotency-Key headers are ambiguous: which one names the attempt? + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"x"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Add("Idempotency-Key", "one") + req.Header.Add("Idempotency-Key", "two") + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("two keys: status %d, want 400", rec.Code) + } +} + +func TestReplayReturnsIdenticalBody(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "повторы", alice.ID) + first := ts.send(alice.token, room.ID, "retry-1", map[string]string{"text": "Привет 👋"}) + ts.expect(first, http.StatusCreated, "") + // The same JSON with other spacing is the same request. + again := ts.send(alice.token, room.ID, "retry-1", ` { "text" : "Привет 👋" } `) + ts.expect(again, http.StatusCreated, "") + if !bytes.Equal(first.raw, again.raw) { + t.Fatalf("replay body differs:\n%s\n%s", first.raw, again.raw) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1", len(page.Items)) + } +} + +func TestReplayWithDifferentBodyIs409(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "конфликт", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "первое"}), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "второе"}), http.StatusConflict, "conflict") +} + +// A replay is a new request: a revoked session cannot fetch the stored result. +func TestReplayRechecksSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "сессия", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusCreated, "") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusNoContent, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") +} + +// The header is checked before access, like the JSON syntax: the answer does +// not depend on the chat, so it reveals nothing about a foreign one. +func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + room := ts.chat(alice.token, "чужой", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + ts.expect(ts.do(http.MethodPost, path, eve.token, map[string]string{"text": "x"}, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found") +} + +// Two applications over the same stores stand for two replicas behind the +// balancer: retries of one key alternate between them and still create one +// message. +func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "реплики", alice.ID) + other := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "00000000000000b2", + }) + engines := []http.Handler{ts.a.Engine, other.Engine} + + const attempts = 10 + type result struct { + status int + body string + instance string + } + results := make([]result, attempts) + var wg sync.WaitGroup + for i := range attempts { + wg.Go(func() { + req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", strings.NewReader(`{"text":"один раз"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Set("Idempotency-Key", "shared") + rec := httptest.NewRecorder() + engines[i%2].ServeHTTP(rec, req) + results[i] = result{rec.Code, rec.Body.String(), rec.Header().Get("X-Instance-Id")} + }) + } + wg.Wait() + instances := map[string]bool{} + for i, r := range results { + if r.status != http.StatusCreated || r.body != results[0].body { + t.Fatalf("attempt %d: %d %s; want 201 %s", i, r.status, r.body, results[0].body) + } + instances[r.instance] = true + } + if len(instances) != 2 { + t.Fatalf("instances %v, want both replicas", instances) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1: %+v", len(page.Items), page.Items) + } +} diff --git a/internal/chat/chat.go b/internal/chat/chat.go index c9281351..e84ff665 100644 --- a/internal/chat/chat.go +++ b/internal/chat/chat.go @@ -28,6 +28,10 @@ const ( // member of, so outsiders cannot probe for existence. var ErrNotFound = apperr.NotFound("chat not found") +// ErrKeyReused answers a send whose Idempotency-Key is already bound to a +// message with a different text. +var ErrKeyReused = apperr.Conflict("Idempotency-Key was already used with a different message") + type Chat struct { ID uuid.UUID Title string diff --git a/internal/chat/cleanup.go b/internal/chat/cleanup.go new file mode 100644 index 00000000..1fd1d050 --- /dev/null +++ b/internal/chat/cleanup.go @@ -0,0 +1,33 @@ +package chat + +import ( + "context" + "log/slog" + "time" +) + +// RunKeyCleanup deletes expired Idempotency-Keys every period until ctx is +// cancelled. Every replica runs it: the deletes skip rows another replica or +// a request holds, so concurrent runs share the work instead of waiting, and +// a failed run is simply retried at the next tick. +func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + deleted, err := svc.PurgeExpiredKeys(ctx) + if err != nil { + if ctx.Err() == nil { + log.Warn("idempotency key cleanup failed", "deleted", deleted, "error", err.Error()) + } + continue + } + if deleted > 0 { + log.Info("idempotency keys purged", "deleted", deleted) + } + } +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go index 9a2c2cc9..b70737bf 100644 --- a/internal/chat/handler.go +++ b/internal/chat/handler.go @@ -2,6 +2,7 @@ package chat import ( "net/http" + "regexp" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -133,12 +134,17 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } + key, err := idempotencyKey(c) + if err != nil { + httpx.Error(c, err) + return + } var req createMessageRequest if err := httpx.DecodeJSON(c, &req); err != nil { httpx.Error(c, err) return } - message, err := h.chats.Send(c.Request.Context(), actor, chatID, req.Text) + message, err := h.chats.Send(c.Request.Context(), actor, chatID, key, req.Text) if err != nil { httpx.Error(c, err) return @@ -165,6 +171,20 @@ func (h *Handler) history(c *gin.Context, actor uuid.UUID) { httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse)) } +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +// idempotencyKey reads the Idempotency-Key that names one send attempt. It +// is part of this endpoint's input contract, so it is checked here, with +// the path and the body, and before access: the answer to a malformed key +// is the same for any chat. Two headers are ambiguous and rejected. +func idempotencyKey(c *gin.Context) (string, error) { + values := c.Request.Header.Values("Idempotency-Key") + if len(values) != 1 || !keyPattern.MatchString(values[0]) { + return "", apperr.Invalid("Idempotency-Key header is required: one value of 1-128 characters A-Z a-z 0-9 . _ : -") + } + return values[0], nil +} + func chatParam(c *gin.Context) (uuid.UUID, error) { id, err := ident.Parse(c.Param("chat_id")) if err != nil { diff --git a/internal/chat/repository.go b/internal/chat/repository.go index 765b1840..83fd6d37 100644 --- a/internal/chat/repository.go +++ b/internal/chat/repository.go @@ -2,6 +2,7 @@ package chat import ( "context" + "time" "github.com/google/uuid" ) @@ -23,7 +24,13 @@ type ChatRepository interface { } type MessageRepository interface { - Append(ctx context.Context, message Message) error + // AppendOnce stores message under the sender's key in its chat, unless + // the key was used there within ttl; it returns the message the key is + // bound to: the given one or the earlier one. + AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error) + // DeleteExpiredKeys removes up to limit keys older than ttl and reports + // how many it removed. + DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) // ListByChat returns up to limit messages ordered by (created_at, id), // strictly after the message identified by after when set. An unknown // chat yields an empty history. diff --git a/internal/chat/service.go b/internal/chat/service.go index 97e09248..b89fdb3d 100644 --- a/internal/chat/service.go +++ b/internal/chat/service.go @@ -3,6 +3,7 @@ package chat import ( "context" "fmt" + "time" "github.com/google/uuid" @@ -22,10 +23,13 @@ type Service struct { chats ChatRepository messages MessageRepository users UserLookup + keyTTL time.Duration } -func NewService(chats ChatRepository, messages MessageRepository, users UserLookup) *Service { - return &Service{chats: chats, messages: messages, users: users} +// NewService builds the chats service; keyTTL is how long a message +// Idempotency-Key stays bound to its message. +func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service { + return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL} } func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) { @@ -62,7 +66,11 @@ func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, l return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil } -func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string) (Message, error) { +// Send stores a message under the sender's Idempotency-Key. Access and the +// text are checked first, on every attempt: a retry is a new request that +// must still be authorised and valid. A key already bound to a message with +// the same text returns that message; with a different text it is a conflict. +func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, key, text string) (Message, error) { if err := s.requireMember(ctx, actor, chatID); err != nil { return Message{}, err } @@ -70,10 +78,34 @@ func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, text string if err != nil { return Message{}, err } - if err := s.messages.Append(ctx, message); err != nil { + stored, err := s.messages.AppendOnce(ctx, message, key, s.keyTTL) + if err != nil { return Message{}, err } - return message, nil + if stored.ID != message.ID && stored.Text != message.Text { + return Message{}, ErrKeyReused + } + return stored, nil +} + +// keyPurgeBatch bounds one delete statement, so the cleanup never holds many +// row locks or runs long. +const keyPurgeBatch = 1000 + +// PurgeExpiredKeys deletes every key older than the retention period, batch +// by batch, and reports how many it deleted. +func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) { + var total int64 + for { + n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL, keyPurgeBatch) + total += n + if err != nil { + return total, err + } + if n < keyPurgeBatch { + return total, nil + } + } } func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) { diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go index 7e9525d8..3cf95f83 100644 --- a/internal/chat/service_test.go +++ b/internal/chat/service_test.go @@ -4,10 +4,13 @@ import ( "context" "errors" "fmt" + "io" + "log/slog" "os" "strings" "sync/atomic" "testing" + "time" "github.com/google/uuid" @@ -67,7 +70,7 @@ func newFixture(t *testing.T) fixture { return u.ID } return fixture{ - svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users), + svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users, time.Hour), users: users, alice: register("alice"), bob: register("bob"), @@ -187,15 +190,15 @@ func TestSendChecksAccessBeforeValidation(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = f.svc.Send(f.ctx, f.eve, c.ID, " ") + _, err = f.svc.Send(f.ctx, f.eve, c.ID, "k1", " ") if !errors.Is(err, chat.ErrNotFound) { t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err) } - _, err = f.svc.Send(f.ctx, f.bob, c.ID, " ") + _, err = f.svc.Send(f.ctx, f.bob, c.ID, "k2", " ") if apperr.KindOf(err) != apperr.KindInvalid { t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err)) } - msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "Привет 👋") + msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "k3", "Привет 👋") if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" { t.Fatalf("Send = %+v, %v", msg, err) } @@ -207,8 +210,8 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) { if err != nil { t.Fatal(err) } - first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "1") - second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "2") + first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "k1", "1") + second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "k2", "2") page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1) if err != nil { @@ -232,3 +235,136 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) { t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err) } } + +func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID { + t.Helper() + c, err := f.svc.Create(f.ctx, members[0], "t", members) + if err != nil { + t.Fatal(err) + } + return c.ID +} + +func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message { + t.Helper() + m, err := f.svc.Send(f.ctx, actor, c, key, text) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestSendReplaysSameBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "привет") + again := f.send(t, f.alice, c, "k", "привет") + if again.ID != first.ID || !again.CreatedAt.Equal(first.CreatedAt) { + t.Fatalf("replay = %+v, want %+v", again, first) + } +} + +func TestSendConflictsOnDifferentBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "первое") + _, err := f.svc.Send(f.ctx, f.alice, c, "k", "другое") + if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err) + } +} + +func TestKeysAreScopedByUserAndChat(t *testing.T) { + f := newFixture(t) + c1 := f.newChat(t, f.alice, f.bob) + c2 := f.newChat(t, f.alice, f.bob) + a1 := f.send(t, f.alice, c1, "k", "текст") + b1 := f.send(t, f.bob, c1, "k", "текст") + a2 := f.send(t, f.alice, c2, "k", "текст") + if a1.ID == b1.ID || a1.ID == a2.ID || b1.ID == a2.ID { + t.Fatalf("one key shared across users or chats: %v %v %v", a1.ID, b1.ID, a2.ID) + } +} + +// A replay checks access again: an outsider presenting a member's key learns +// nothing and the key stays bound to the member's message. +func TestSendChecksAccessBeforeKey(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "секрет") + if _, err := f.svc.Send(f.ctx, f.eve, c, "k", "секрет"); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err) + } + if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID { + t.Fatalf("key rebound after the outsider: %v, want %v", again.ID, first.ID) + } +} + +func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "текст") + if _, err := f.svc.Send(f.ctx, f.alice, c, "k", " "); apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err)) + } +} + +func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + const expired = 2500 + _, err := db.Pool.Exec(f.ctx, ` + WITH m AS ( + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $1, gen_random_uuid(), $2, 'old', now() FROM generate_series(1, $3) + RETURNING id) + INSERT INTO message_idempotency_keys (user_id, chat_id, key, message_id, created_at) + SELECT $2, $1, id::text, id, now() - interval '2 hours' FROM m`, c, f.alice, expired) + if err != nil { + t.Fatal(err) + } + f.send(t, f.alice, c, "fresh", "новое") + deleted, err := f.svc.PurgeExpiredKeys(f.ctx) + if err != nil || deleted != expired { + t.Fatalf("PurgeExpiredKeys = %d, %v; want %d", deleted, err, expired) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want the fresh one", left, err) + } +} + +func TestRunKeyCleanupPurgesAndStops(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "old", "старое") + if _, err := db.Pool.Exec(f.ctx, `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(f.ctx) + stopped := make(chan struct{}) + go func() { + chat.RunKeyCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), f.svc, 50*time.Millisecond) + close(stopped) + }() + deadline := time.Now().Add(2 * time.Second) + for { + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil { + t.Fatal(err) + } + if left == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("expired key was not purged within 2s") + } + time.Sleep(20 * time.Millisecond) + } + cancel() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("RunKeyCleanup did not return after cancel") + } +} diff --git a/internal/chat/store/export_test.go b/internal/chat/store/export_test.go new file mode 100644 index 00000000..528730c4 --- /dev/null +++ b/internal/chat/store/export_test.go @@ -0,0 +1,9 @@ +package store + +// SetAfterClaim installs a hook that runs between the claiming statement and +// the replay lookup of AppendOnce; it returns a function restoring the old one. +func SetAfterClaim(hook func()) (restore func()) { + old := afterClaim + afterClaim = hook + return func() { afterClaim = old } +} diff --git a/internal/chat/store/idempotency_test.go b/internal/chat/store/idempotency_test.go new file mode 100644 index 00000000..72f8714e --- /dev/null +++ b/internal/chat/store/idempotency_test.go @@ -0,0 +1,252 @@ +package store_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" +) + +const keyTTL = time.Hour + +// expire makes every stored key older than keyTTL. +func expire(t *testing.T) { + t.Helper() + if _, err := db.Pool.Exec(context.Background(), + `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } +} + +func (f fixture) appendOnce(t *testing.T, m chat.Message, key string) chat.Message { + t.Helper() + got, err := f.messages.AppendOnce(f.ctx, m, key, keyTTL) + if err != nil { + t.Fatal(err) + } + return got +} + +func (f fixture) count(t *testing.T, chatID uuid.UUID) int { + t.Helper() + got, err := f.messages.ListByChat(f.ctx, chatID, nil, 1000) + if err != nil { + t.Fatal(err) + } + return len(got) +} + +func keyTarget(t *testing.T, key string) uuid.UUID { + t.Helper() + var id uuid.UUID + if err := db.Pool.QueryRow(context.Background(), + `SELECT message_id FROM message_idempotency_keys WHERE key = $1`, key).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +func TestAppendOnceInsertsThenReplays(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, first, "k1"); got.ID != first.ID { + t.Fatalf("first append returned %v, want the new message %v", got.ID, first.ID) + } + retry := f.newMessage(t, c.ID, alice) + got := f.appendOnce(t, retry, "k1") + if got.ID != first.ID || !got.CreatedAt.Equal(first.CreatedAt) || got.Text != first.Text || got.SenderID != alice { + t.Fatalf("replay returned %+v, want the stored %+v", got, first) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} + +func TestAppendOnceTakesOverExpiredKey(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + next := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, next, "k1"); got.ID != next.ID { + t.Fatalf("expired key returned %v, want the new message %v", got.ID, next.ID) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } + if target := keyTarget(t, "k1"); target != next.ID { + t.Fatalf("key points to %v, want %v", target, next.ID) + } +} + +// A cleaner batch holds the expired key row; the request waits for it and, +// once the row is gone, inserts its message under the now free key. +func TestAppendOnceWaitsForCleanerThenInserts(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(f.ctx) + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + + next := f.newMessage(t, c.ID, alice) + type result struct { + m chat.Message + err error + } + done := make(chan result, 1) + go func() { + m, err := f.messages.AppendOnce(f.ctx, next, "k1", keyTTL) + done <- result{m, err} + }() + select { + case r := <-done: + t.Fatalf("AppendOnce did not wait for the locked key: %+v", r) + case <-time.After(300 * time.Millisecond): + } + if _, err := tx.Exec(f.ctx, `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil { + t.Fatal(err) + } + if err := tx.Commit(f.ctx); err != nil { + t.Fatal(err) + } + select { + case r := <-done: + if r.err != nil || r.m.ID != next.ID { + t.Fatalf("after the cleaner: %+v, %v; want the new message %v", r.m, r.err, next.ID) + } + case <-time.After(5 * time.Second): + t.Fatal("AppendOnce did not finish after the cleaner committed") + } +} + +// A request holding a key row must not make the cleaner wait: it skips the +// row and deletes it on a later run. +func TestCleanerSkipsRowLockedByRequest(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + start := time.Now() + n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000) + if err != nil || n != 0 { + t.Fatalf("cleaner over a locked row: deleted %d, %v; want 0", n, err) + } + if waited := time.Since(start); waited > time.Second { + t.Fatalf("cleaner waited %v for the locked row", waited) + } + if err := tx.Rollback(f.ctx); err != nil { + t.Fatal(err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000); err != nil || n != 1 { + t.Fatalf("cleaner after release: deleted %d, %v; want 1", n, err) + } +} + +// The key is alive when the claim runs but deleted before the replay lookup, +// as by a cleaner whose clock is a moment later at the TTL boundary. The +// operation starts over and stores the new message. +func TestAppendOnceRetriesWhenKeyVanishes(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + var once sync.Once + restore := store.SetAfterClaim(func() { + once.Do(func() { + if _, err := db.Pool.Exec(context.Background(), `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil { + t.Error(err) + } + }) + }) + defer restore() + next := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, next, "k1"); got.ID != next.ID { + t.Fatalf("got %v, want the new message %v", got.ID, next.ID) + } +} + +func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + for _, k := range []string{"a", "b", "c"} { + f.appendOnce(t, f.newMessage(t, c.ID, alice), k) + } + expire(t) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "fresh") + + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 2 { + t.Fatalf("first batch deleted %d, %v; want 2", n, err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 1 { + t.Fatalf("second batch deleted %d, %v; want 1", n, err) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want only the fresh one", left, err) + } + if n := f.count(t, c.ID); n != 4 { + t.Fatalf("cleanup removed messages: %d left, want 4", n) + } +} + +// Every request brings its own new message under one key; exactly one is +// stored and all of them get it back. This also proves the key row and the +// message it references are written by one statement. +func TestConcurrentAppendOnceSameKeyStoresOneMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const requests = 20 + ids := make(chan uuid.UUID, requests) + var wg sync.WaitGroup + for range requests { + m := f.newMessage(t, c.ID, alice) + wg.Go(func() { + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + ids <- got.ID + }) + } + wg.Wait() + close(ids) + seen := map[uuid.UUID]int{} + for id := range ids { + seen[id]++ + } + if len(seen) != 1 { + t.Fatalf("requests got %d different messages, want 1: %v", len(seen), seen) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index e3634a38..d93f6592 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -12,6 +12,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -117,14 +118,90 @@ func NewMessages(pool *pgxpool.Pool) *Messages { return &Messages{pool: pool} } -func (r *Messages) Append(ctx context.Context, m chat.Message) error { - _, err := r.pool.Exec(ctx, - `INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES ($1, $2, $3, $4, $5)`, - m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt) - if err != nil { - return fmt.Errorf("insert message: %w", postgres.Classify(err)) +// afterClaim runs between the claiming statement and the replay lookup of +// AppendOnce; tests use it to delete the key in that window. +var afterClaim func() + +// claimSQL stores the key and the message in one statement. A key older than +// the retention period is taken over for the new message. The key row is +// inserted first: when it already exists and is still valid, claimed is +// empty and so is the message insert. One statement holds the key's lock +// only while the server executes it, never across a round trip to a replica +// that might die in between. +const claimSQL = ` + WITH claimed AS ( + INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET message_id = EXCLUDED.message_id, created_at = now() + WHERE k.created_at < now() - $5::interval + RETURNING message_id) + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $2, message_id, $1, $6, $7 FROM claimed` + +// AppendOnce stores m under key unless the sender already used the key in +// this chat within ttl; it returns the message the key is bound to, m itself +// or the earlier one. Deciding whether the earlier message answers the same +// request is the caller's job. +// +// The lookup of an existing key is a second statement: within the claiming +// one, a plain SELECT would see the snapshot taken before a concurrent +// request committed the key. Between the two statements the cleaner may +// delete a key at the ttl boundary; the operation then starts over once, +// which stores m under the now free key. +func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { + for range 2 { + tag, err := r.pool.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) + } + if tag.RowsAffected() == 1 { + return m, nil + } + if afterClaim != nil { + afterClaim() + } + var stored chat.Message + err = r.pool.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) + } + return stored, nil } - return nil + return chat.Message{}, errors.New("idempotency key vanished twice between claim and lookup") +} + +// DeleteExpiredKeys removes up to limit keys older than ttl. SKIP LOCKED lets +// replicas cleaning at the same time, and requests holding a key, pass each +// other instead of waiting. +func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) { + tag, err := r.pool.Exec(ctx, ` + DELETE FROM message_idempotency_keys + WHERE (user_id, chat_id, key) IN ( + SELECT user_id, chat_id, key FROM message_idempotency_keys + WHERE created_at < now() - $1::interval + ORDER BY created_at + LIMIT $2 + FOR UPDATE SKIP LOCKED)`, + interval(ttl), limit) + if err != nil { + return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err)) + } + return tag.RowsAffected(), nil +} + +// interval renders d for a ::interval parameter. +func interval(d time.Duration) string { + return fmt.Sprintf("%d milliseconds", d.Milliseconds()) } // ListByChat is the history query; scripts/explain-history.sql runs the same diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go index 5b005ea7..8e843d23 100644 --- a/internal/chat/store/postgres_test.go +++ b/internal/chat/store/postgres_test.go @@ -8,6 +8,7 @@ import ( "os" "sync" "testing" + "time" "github.com/google/uuid" @@ -81,6 +82,12 @@ func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message return m } +// appendErr stores m under a key of its own, as every send now carries one. +func (f fixture) appendErr(m chat.Message) error { + _, err := f.messages.AppendOnce(f.ctx, m, m.ID.String(), time.Hour) + return err +} + func TestChatCreateThenGetForMember(t *testing.T) { f := newFixture(t) alice, bob := f.mustUser(t), f.mustUser(t) @@ -199,7 +206,7 @@ func TestAppendOutOfOrderIsListedByID(t *testing.T) { c := f.mustChat(t, alice) first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice) for _, m := range []chat.Message{third, first, second} { - if err := f.messages.Append(f.ctx, m); err != nil { + if err := f.appendErr(m); err != nil { t.Fatal(err) } } @@ -221,13 +228,13 @@ func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) { for range total { m := f.newMessage(t, c.ID, alice) expected = append(expected, m.ID) - if err := f.messages.Append(f.ctx, m); err != nil { + if err := f.appendErr(m); err != nil { t.Fatal(err) } } // Another chat's history must not leak into this one. other := f.mustChat(t, alice) - if err := f.messages.Append(f.ctx, f.newMessage(t, other.ID, alice)); err != nil { + if err := f.appendErr(f.newMessage(t, other.ID, alice)); err != nil { t.Fatal(err) } for _, limit := range []int{1, 7, 100} { @@ -269,7 +276,7 @@ func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) { gap := ident.New() // never stored, but sorts between a and b b := f.newMessage(t, c.ID, alice) for _, m := range []chat.Message{a, b} { - if err := f.messages.Append(f.ctx, m); err != nil { + if err := f.appendErr(m); err != nil { t.Fatal(err) } } @@ -291,7 +298,7 @@ func TestConcurrentAppendsKeepEveryMessage(t *testing.T) { for range clients { wg.Go(func() { for range perClient { - if err := f.messages.Append(f.ctx, f.newMessage(t, c.ID, alice)); err != nil { + if err := f.appendErr(f.newMessage(t, c.ID, alice)); err != nil { t.Error(err) } } diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go index 49dc627a..76daaf7c 100644 --- a/internal/platform/postgres/postgres_test.go +++ b/internal/platform/postgres/postgres_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net" "os" "testing" @@ -15,6 +16,7 @@ import ( "messenger/internal/platform/apperr" "messenger/internal/platform/postgres" "messenger/internal/testenv" + "messenger/migrations" ) var db *testenv.DB @@ -54,10 +56,14 @@ func TestMigrateIsRepeatable(t *testing.T) { if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil { t.Fatal(err) } - if version != 1 { - t.Fatalf("schema version = %d, want 1", version) + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) } - for _, table := range []string{"users", "chats", "chat_members", "messages"} { + if version != int64(len(files)) { + t.Fatalf("schema version = %d, want %d (one per migration file)", version, len(files)) + } + for _, table := range []string{"users", "chats", "chat_members", "messages", "message_idempotency_keys"} { var exists bool if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil { t.Fatal(err) diff --git a/migrations/00002_message_idempotency_keys.sql b/migrations/00002_message_idempotency_keys.sql new file mode 100644 index 00000000..2f35f086 --- /dev/null +++ b/migrations/00002_message_idempotency_keys.sql @@ -0,0 +1,25 @@ +-- +goose Up +-- One row per Idempotency-Key a user has sent to one chat. The contract scope +-- is user + method + chat path + key; only POST /chats/{chat_id}/messages +-- accepts keys, so (user_id, chat_id) already fixes the method and the path. +-- The primary key is what makes concurrent retries through different +-- replicas safe: the second insert of a key waits for the first one's commit. +-- The row references the message instead of storing a response: messages +-- are immutable, so a replay is answered from the source of truth. +-- The table only adds to the schema: the previous API version keeps working +-- while a rolling update runs this migration. +CREATE TABLE message_idempotency_keys ( + user_id uuid NOT NULL, + chat_id uuid NOT NULL, + key text NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key), + FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id) +); + +-- Serves the periodic removal of keys older than the retention period. +CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at); + +-- +goose Down +DROP TABLE message_idempotency_keys; diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go index 5f915761..62bd2d5e 100644 --- a/tools/concurrency/client.go +++ b/tools/concurrency/client.go @@ -8,6 +8,8 @@ import ( "net/http" "net/url" "time" + + "github.com/google/uuid" ) // client is a minimal typed HTTP client for the messenger API. @@ -33,6 +35,21 @@ func (c *client) post(path, token string, body, out any) error { return c.do(req, token, http.StatusCreated, out) } +// postMessage sends one message under a fresh Idempotency-Key. +func (c *client) postMessage(chatID, token, text string, out any) error { + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", uuid.NewString()) + return c.do(req, token, http.StatusCreated, out) +} + // login opens a session with Basic credentials and returns its bearer token. func (c *client) login(username, password string) (string, error) { req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go index 0ed1e54e..eea5e108 100644 --- a/tools/concurrency/main.go +++ b/tools/concurrency/main.go @@ -174,8 +174,7 @@ func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (m <-start for m := range perClient { var msg chat.MessageResponse - body := map[string]string{"text": fmt.Sprintf("client %d message %d", n, m)} - err := own.post("/api/v1/chats/"+chatID+"/messages", token, body, &msg) + err := own.postMessage(chatID, token, fmt.Sprintf("client %d message %d", n, m), &msg) mu.Lock() if err != nil { failed++ -- 2.54.0 From c5cdcb337777c08fbe34c36139a94a6ed6193074 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 17:20:38 +0300 Subject: [PATCH 04/21] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=BD=D1=8F=D1=82?= =?UTF-8?q?=D1=8C=20API=20=D0=B2=20=D0=BD=D0=B5=D1=81=D0=BA=D0=BE=D0=BB?= =?UTF-8?q?=D1=8C=D0=BA=D0=B8=D1=85=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=D1=85=20=D0=B7=D0=B0=20HAProxy=20=D0=B8=20=D0=B4=D0=BE?= =?UTF-8?q?=D0=B1=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20=D1=81=D0=BA=D1=80=D0=B8?= =?UTF-8?q?=D0=BF=D1=82=D1=8B=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=B0=20?= =?UTF-8?q?=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8=20=D0=B8=20rolling?= =?UTF-8?q?=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 1 + Makefile | 11 +++- compose.yaml | 102 ++++++++++++++++++++++++++++++++++---- infra/haproxy/haproxy.cfg | 83 +++++++++++++++++++++++++++++++ scripts/api-instances.sh | 34 +++++++++++++ scripts/instances.sh | 11 ++++ scripts/rollout.sh | 44 ++++++++++++++++ scripts/stop-one.cmd | 5 ++ scripts/stop-one.sh | 31 ++++++++++++ 9 files changed, 312 insertions(+), 10 deletions(-) create mode 100644 infra/haproxy/haproxy.cfg create mode 100755 scripts/api-instances.sh create mode 100755 scripts/instances.sh create mode 100755 scripts/rollout.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh diff --git a/.gitattributes b/.gitattributes index 1af1a8fe..2cc26e76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,5 +2,6 @@ # checkouts with core.autocrlf=true. *.sh text eol=lf *.conf text eol=lf +*.cfg text eol=lf *.sql text eol=lf *.cmd text eol=crlf diff --git a/Makefile b/Makefile index 63093d80..4c74f9de 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ test-failover: # --------------------------------------------------------------------------- EVIDENCE_DIR ?= evidence -.PHONY: build vet unit secrets up down destroy logs concurrency explain +.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout build: go build -o bin/server ./cmd/server @@ -63,6 +63,15 @@ destroy: logs: docker compose logs +# Which container answers under which X-Instance-Id. +instances: + sh scripts/instances.sh + +# Replaces every API replica with one built from the current sources, without +# downtime; restore a replica stopped by stop-one with `docker compose start api`. +rollout: + sh scripts/rollout.sh + concurrency: go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" diff --git a/compose.yaml b/compose.yaml index 9228acc9..ccaa6ab9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,13 +1,51 @@ -# Local stand: the API over PostgreSQL (users, chats, messages) and Valkey -# (sessions). Only the API is published, on loopback. Credentials are files in +# Local stand: HAProxy in front of interchangeable API replicas over shared +# PostgreSQL (users, chats, messages, idempotency keys) and Valkey (sessions). +# Only the balancer is published, on loopback. Credentials are files in # ./secrets created by `make secrets`; named volumes keep the data across # restarts, and only `make destroy` removes them. +# +# Two networks: edge joins the balancer and the API, data joins the API, the +# migrations and the stores and has no way out. The balancer cannot reach the +# stores, and the stores cannot reach anything outside. +name: lab3 + services: + lb: + image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + environment: + # 1 silences the log line of every successful request during load runs. + LB_QUIET: ${LB_QUIET:-0} + # One HAProxy thread per CPU of the limit below. + LB_THREADS: ${LB_THREADS:-1} + ports: + # Loopback only; override with PUBLISH=127.0.0.1:28080 when 8080 is busy. + - "${PUBLISH:-127.0.0.1:8080}:8080" + # Read-only statistics page: state and checks of every replica. + - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" + networks: + - edge + healthcheck: + # Through the balancer itself: healthy once it can route to a ready + # replica, so `compose up --wait` means "ready through the entry point". + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + deploy: + resources: + limits: + cpus: "${LB_CPUS:-0.5}" + memory: 128M + api: build: . image: mtusi-messenger:lab3 - # Fixed name for docker logs/inspect in the report. - container_name: mtusi-messenger-lab3 + # No container_name and no ports: the service scales, and clients reach + # the replicas only through the balancer. env_file: - path: .env required: false @@ -15,9 +53,20 @@ services: DATABASE_URL_FILE: /run/secrets/database_url SESSION_STORE_URL_FILE: /run/secrets/session_store_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + # Above the balancer's detection time of 1.5 s (haproxy.cfg). + APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-2s} + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + # About 90 % of the memory limit below; Go does not read the cgroup limit. + GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} + # Empty disables the internal expvar listener; the benchmark sets :9090. + APP_DEBUG_ADDR: ${APP_DEBUG_ADDR:-} secrets: - database_url - session_store_url + networks: + - edge + - data depends_on: migrate: condition: service_completed_successfully @@ -25,12 +74,16 @@ services: condition: service_healthy valkey: condition: service_healthy - # Loopback only; override with PUBLISH=127.0.0.1:18080 when 8080 is busy. - ports: - - "${PUBLISH:-127.0.0.1:8080}:8080" - stop_grace_period: 10s + # Drain delay 2 s + shutdown timeout 8 s fit before Docker sends SIGKILL. + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-256M} - # One-shot: applies the schema before the API starts, then exits. + # One-shot: applies the schema once before the replicas start, then exits. migrate: image: mtusi-messenger:lab3 entrypoint: ["/usr/local/bin/migrate"] @@ -38,15 +91,26 @@ services: DATABASE_URL_FILE: /run/secrets/database_url secrets: - database_url + networks: + - data depends_on: postgres: condition: service_healthy restart: "no" healthcheck: + # The image's check probes the API port, which migrate never opens. disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 + # Ten replicas with ten connections each fit during a rolling update of + # five, plus migrate and psql sessions. + command: ["postgres", "-c", "max_connections=150"] environment: POSTGRES_DB: messenger POSTGRES_USER: messenger @@ -57,6 +121,8 @@ services: # below /var/lib/postgresql, so the volume is mounted on the parent. volumes: - pgdata:/var/lib/postgresql + networks: + - data healthcheck: # Over TCP: during first initialisation the image runs a temporary # server on the Unix socket only, which must not count as ready. @@ -68,6 +134,11 @@ services: # A clean shutdown writes a checkpoint instead of leaving crash recovery # to the next start. stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEMORY:-512M} valkey: image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 @@ -77,6 +148,8 @@ services: volumes: - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro - valkeydata:/data + networks: + - data healthcheck: # valkey-cli exits 0 even on NOAUTH, so the reply itself is checked. test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"] @@ -85,11 +158,22 @@ services: retries: 10 start_period: 5s stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.5" + # maxmemory 256mb (valkey.conf) plus the server's own overhead. + memory: 320M volumes: pgdata: valkeydata: +networks: + edge: {} + data: + internal: true + secrets: database_url: file: ./secrets/database_url diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..861a97d8 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,83 @@ +# The single entry point: round-robin over the API replicas found through +# Docker DNS, with active readiness checks and no client stickiness. Replicas +# are interchangeable: sessions live in Valkey, everything else in PostgreSQL. + +global + log stdout format raw local0 info + maxconn 20000 + # One thread per CPU of the container limit; by default HAProxy would + # start one per CPU of the host and fight the CPU quota. + nbthread "${LB_THREADS}" + +defaults + mode http + log global + option dontlognull + # Inside the Docker network a replica accepts or refuses at once; a + # second means it is gone. + timeout connect 1s + timeout client 30s + # Above the API's own 5 s request deadline: a slow request ends in the + # API's 503 with its cause logged, and this timeout only cuts off a + # replica that hangs entirely. + timeout server 7s + timeout http-request 10s + timeout http-keep-alive 10s + timeout check 2s + # A request is re-sent, to another replica, only when the connection + # failed and so nothing reached the API. A timeout or an empty response + # may follow a committed write, and POST /users or /chats must never be + # repeated behind the client's back (contract, idempotency section). + retries 2 + option redispatch 1 + retry-on conn-failure + # Errors produced by the balancer itself use the contract's envelope. They + # carry X-Request-Id but no X-Instance-Id: no replica answered. + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + +# Docker's embedded DNS: the service name `api` resolves to every running +# replica. Re-resolved every second, so scaled, restarted or replaced +# replicas join and stopped ones leave without a reload. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + hold obsolete 0s + accepted_payload_size 8192 + +frontend entry + bind :8080 + option httplog + # 32 hex characters, like the request ids of the API. + unique-id-format %[uuid(4),regsub(-,,g)] + # The access log names the replica that answered and its request id, which + # is the key to the replica's own log line. + capture response header X-Instance-Id len 32 + capture response header X-Request-Id len 40 + http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 } + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + # Detection: a replica that fails readiness is out after the next check + # (inter 1 s) plus one fast re-check (fastinter 0.5 s), so within 1.5 s. + # A stopping API keeps serving for APP_DRAIN_DELAY (2 s) after readiness + # turns 503, longer than this, and only then closes its listener. Back in + # rotation after two good checks (rise 2), about 2 s. + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 + # Ten slots: up to five replicas, doubled while docker rollout replaces them. + server-template api 10 api:8080 + +listen stats + bind :8404 + no log + stats enable + stats uri / + stats refresh 2s diff --git a/scripts/api-instances.sh b/scripts/api-instances.sh new file mode 100755 index 00000000..4a90d4f1 --- /dev/null +++ b/scripts/api-instances.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Lists the API containers of this compose project, one per line: +# +# The alias is what the process sends as X-Instance-Id. It is random per +# process, so it is asked from the process itself, over the same wget the +# Docker healthcheck uses, from inside the container. A container that is not +# running has no alias and shows "-". The replicas are asked in parallel: the +# failover hook that relies on this list must return within 10 seconds. +set -eu +cd "$(dirname "$0")/.." + +ids=$(docker compose ps -aq api) +[ -n "$ids" ] || exit 0 +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +docker inspect --format '{{.Id}} {{.Name}} {{.State.Status}} {{.Image}}' $ids >"$tmp/containers" +while read -r id name state image; do + if [ "$state" = running ]; then + # wget prints the response headers on stderr; the body is dropped. + (docker exec "$id" wget -S -q -O - http://127.0.0.1:8080/health/live 2>&1 >/dev/null | + sed -n 's/^ *[Xx]-[Ii]nstance-[Ii]d: *//p' | tr -d '\r' | head -n 1 >"$tmp/$id") & + fi +done <"$tmp/containers" +wait + +while read -r id name state image; do + alias=- + if [ -s "$tmp/$id" ]; then + alias=$(cat "$tmp/$id") + fi + image=${image#sha256:} + printf '%s %s %s %s %s\n' "$(printf '%s' "$id" | cut -c1-12)" "${name#/}" "$state" "$alias" "$(printf '%s' "$image" | cut -c1-12)" +done <"$tmp/containers" diff --git a/scripts/instances.sh b/scripts/instances.sh new file mode 100755 index 00000000..52150a3f --- /dev/null +++ b/scripts/instances.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Shows which container answers under which X-Instance-Id: the proof on the +# defense that the header values belong to different running processes, and +# which container a failover test actually stopped. +set -eu +cd "$(dirname "$0")/.." + +printf '%-12s %-14s %-8s %-16s %s\n' CONTAINER NAME STATE X-INSTANCE-ID IMAGE +sh scripts/api-instances.sh | while read -r id name state alias image; do + printf '%-12s %-14s %-8s %-16s %s\n' "$id" "$name" "$state" "$alias" "$image" +done diff --git a/scripts/rollout.sh b/scripts/rollout.sh new file mode 100755 index 00000000..1a223a83 --- /dev/null +++ b/scripts/rollout.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Rolling update of the API without downtime, with docker rollout +# (https://github.com/wowu/docker-rollout, MIT): it starts as many new +# replicas as are running, waits until all of them are healthy, then stops +# and removes the old ones. If a new replica does not become healthy it +# removes the new ones and leaves the old ones serving, so the stand is never +# left with a mix of versions. +# +# The migrations of the new version run first, once, while the old replicas +# serve: they only add to the schema. docker rollout scales with `compose up`, +# which also starts the dependencies of the API and would rerun the old, +# exited migrate container. Recreating it beforehand from the new image, +# without starting it, makes that `up` run the new migrations exactly once, +# before any new replica starts. +set -eu +cd "$(dirname "$0")/.." + +version=v0.14 +checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b +tool=bin/docker-rollout + +if ! printf '%s %s\n' "$checksum" "$tool" | sha256sum -c - >/dev/null 2>&1; then + mkdir -p bin + curl -fsSL -o "$tool.download" "https://github.com/wowu/docker-rollout/releases/download/$version/docker-rollout" + if ! printf '%s %s\n' "$checksum" "$tool.download" | sha256sum -c - >/dev/null 2>&1; then + rm -f "$tool.download" + echo "docker-rollout $version: checksum mismatch, not used" >&2 + exit 1 + fi + mv "$tool.download" "$tool" +fi + +docker compose build api +docker compose up --no-start --no-deps --force-recreate migrate +# Run through sh: Windows cannot use the script as a docker CLI plugin. +if ! sh "$tool" rollout --wait-after-healthy 4 api; then + # A failed migration stops the scale-up after the new containers were + # created but before they started; remove them for a clean next attempt. + created=$(docker compose ps -aq --status created api) + if [ -n "$created" ]; then + docker rm $created >/dev/null + fi + exit 1 +fi diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..76322518 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/stop-one.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0stop-one.sh" +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..d2730e94 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Failover hook of `make test-failover` (tests/resilience.py): stops the one +# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it +# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps +# serving for its drain delay while HAProxy takes it out of rotation, then +# exits; `docker stop` returns once it has. Nothing else is touched, and an +# alias that does not name exactly one running replica stops nothing. +# Restore the replica with: docker compose start api +set -eu +cd "$(dirname "$0")/.." + +target=${TARGET_INSTANCE_ID:-} +case "$target" in +'' | *[!0-9a-f]*) + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 + ;; +esac +if [ ${#target} -ne 16 ]; then + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 +fi + +matches=$(sh scripts/api-instances.sh | awk -v t="$target" '$3 == "running" && $4 == t { print $1 }') +count=$(printf '%s\n' "$matches" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo "expected one running API replica with X-Instance-Id $target, found $count" >&2 + exit 2 +fi +docker stop "$matches" >/dev/null +echo "stopped API container $matches (X-Instance-Id $target)" -- 2.54.0 From b2973405f3b7555cb2857676dd386144ee3fc117 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 17:44:59 +0300 Subject: [PATCH 05/21] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B3=D0=B5=D0=BD=D0=B5=D1=80=D0=B0=D1=82=D0=BE?= =?UTF-8?q?=D1=80=20=D0=BD=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8,=20?= =?UTF-8?q?=D1=81=D0=B1=D0=BE=D1=80=20=D0=BC=D0=B5=D1=82=D1=80=D0=B8=D0=BA?= =?UTF-8?q?=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=20=D0=B8=20=D1=81=D1=86?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=D1=80=D0=B8=D0=B8=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE=D0=B2,=20ro?= =?UTF-8?q?lling=20update=20=D0=B8=20=D1=82=D1=80=D0=B5=D0=BA=D0=B0=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/app.yml | 7 +- .gitignore | 2 + Dockerfile | 29 ++- Makefile | 23 +- cmd/server/main.go | 38 +++ compose.yaml | 19 ++ internal/server/server.go | 21 ++ internal/server/server_test.go | 24 ++ requirements-bench.txt | 2 + scripts/bench/failover.sh | 61 +++++ scripts/bench/lib.sh | 57 +++++ scripts/bench/plot.py | 253 ++++++++++++++++++++ scripts/bench/rollout-under-load.sh | 47 ++++ scripts/bench/run-series.sh | 58 +++++ scripts/bench/series/A.conf | 6 + scripts/bench/series/Aprime.conf | 5 + scripts/bench/series/B.conf | 7 + scripts/bench/series/calibrate.conf | 4 + scripts/restart.sh | 3 +- tools/concurrency/client.go | 36 +++ tools/concurrency/main.go | 117 ++++++++- tools/concurrency/main_test.go | 28 ++- tools/loadgen/client.go | 159 +++++++++++++ tools/loadgen/main.go | 352 ++++++++++++++++++++++++++++ tools/loadgen/main_test.go | 104 ++++++++ tools/loadgen/run.go | 54 +++++ tools/loadgen/scrape.go | 142 +++++++++++ tools/loadgen/seed.go | 127 ++++++++++ tools/loadgen/stats.go | 196 ++++++++++++++++ tools/loadgen/stats_test.go | 104 ++++++++ 30 files changed, 2067 insertions(+), 18 deletions(-) create mode 100644 requirements-bench.txt create mode 100755 scripts/bench/failover.sh create mode 100644 scripts/bench/lib.sh create mode 100755 scripts/bench/plot.py create mode 100755 scripts/bench/rollout-under-load.sh create mode 100755 scripts/bench/run-series.sh create mode 100644 scripts/bench/series/A.conf create mode 100644 scripts/bench/series/Aprime.conf create mode 100644 scripts/bench/series/B.conf create mode 100644 scripts/bench/series/calibrate.conf create mode 100644 tools/loadgen/client.go create mode 100644 tools/loadgen/main.go create mode 100644 tools/loadgen/main_test.go create mode 100644 tools/loadgen/run.go create mode 100644 tools/loadgen/scrape.go create mode 100644 tools/loadgen/seed.go create mode 100644 tools/loadgen/stats.go create mode 100644 tools/loadgen/stats_test.go diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index a27b1206..936f2ec4 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -36,7 +36,12 @@ jobs: run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt - name: Persistence across a restart of the API and both stores run: make test-persistence PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/restart.sh | tee evidence-ci/persistence.txt - - name: Concurrency check + - name: Failover of one replica behind the balancer + run: | + make test-failover PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/stop-one.sh | tee evidence-ci/failover.txt + docker compose start api + docker compose up -d --wait + - name: Concurrency and shared idempotency across replicas run: go run ./tools/concurrency --base-url "$BASE_URL" --output evidence-ci/concurrency.json - name: Container logs if: always() diff --git a/.gitignore b/.gitignore index b5a71896..94c5da0a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Raw output of local benchmark runs; curated results go to evidence/. +bench-out/ diff --git a/Dockerfile b/Dockerfile index 571f6bb7..f32eecf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,34 @@ # Build stage: compile static binaries with the exact toolchain pinned in go.mod. +# Modules live in their own layer, rebuilt only when go.mod or go.sum change. +# Only the Go sources are copied after it, and the compiler's cache persists +# between builds: a change elsewhere in the repository does not trigger a full +# recompilation, which matters when a rolling update rebuilds the image. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download && go mod verify -COPY . . -RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate +COPY cmd ./cmd +COPY internal ./internal +COPY migrations ./migrations +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Runtime stage: minimal image, unprivileged user, the server is PID 1. The -# same image runs the one-shot migrate service with its own entrypoint. +# Load generator of the experiments, an image of its own: the API image does +# not carry it. Built only for `docker compose --profile bench`. +FROM build AS loadgen-build +COPY tools ./tools +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen + +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen +RUN adduser -D -H -u 10001 app +COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen +USER app +ENTRYPOINT ["/usr/local/bin/loadgen"] + +# Runtime stage, the last one so that `build: .` builds it: minimal image, +# unprivileged user, the server is PID 1. The same image runs the one-shot +# migrate service with its own entrypoint. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app COPY --from=build /out/server /out/migrate /usr/local/bin/ diff --git a/Makefile b/Makefile index 4c74f9de..39794250 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ test-failover: # --------------------------------------------------------------------------- EVIDENCE_DIR ?= evidence -.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout +.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout bench plot failover-load rollout-load build: go build -o bin/server ./cmd/server @@ -72,6 +72,27 @@ instances: rollout: sh scripts/rollout.sh +# Track 1, the cost of scaling: one series on fresh stands, then the charts. +# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800 +SERIES ?= scripts/bench/series/A.conf +REPEATS ?= 3 +bench: + sh scripts/bench/run-series.sh $(SERIES) $(REPEATS) + +# Needs matplotlib from requirements-bench.txt: make plot PYTHON=. +plot: + $(PYTHON) scripts/bench/plot.py evidence/track1/raw evidence/track1 + +# Level 4 under load: one replica stopped (KIND=stop) or killed (KIND=kill), +# and the rolling update. +KIND ?= stop +REPLICA ?= 1 +failover-load: + sh scripts/bench/failover.sh $(KIND) $(REPLICA) + +rollout-load: + sh scripts/bench/rollout-under-load.sh + concurrency: go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" diff --git a/cmd/server/main.go b/cmd/server/main.go index f3cbafcf..72e2440f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,7 @@ package main import ( "context" + "expvar" "fmt" "log/slog" "net" @@ -13,6 +14,8 @@ import ( "syscall" "time" + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/app" "messenger/internal/chat" "messenger/internal/platform/config" @@ -80,10 +83,45 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) var background sync.WaitGroup defer background.Wait() // before the deferred pool.Close background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if cfg.DebugAddr != "" { + publishDebugVars(alias, pool) + debugLn, err := net.Listen("tcp", cfg.DebugAddr) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) + } + background.Go(func() { + if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { + log.Error("debug listener stopped", "error", err.Error()) + } + }) + } return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) } +// publishDebugVars exposes, next to the runtime's memstats, what the scaling +// study needs from a replica: who it is, how many CPUs Go uses inside the +// container's limit, and how often requests had to wait for a pooled +// database connection. +func publishDebugVars(alias string, pool *pgxpool.Pool) { + expvar.NewString("instance").Set(alias) + expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) })) + expvar.Publish("pgxpool", expvar.Func(func() any { + s := pool.Stat() + return map[string]any{ + "acquire_count": s.AcquireCount(), + "acquire_duration_ms": s.AcquireDuration().Milliseconds(), + "acquired_conns": s.AcquiredConns(), + "idle_conns": s.IdleConns(), + "total_conns": s.TotalConns(), + "max_conns": s.MaxConns(), + "empty_acquire_count": s.EmptyAcquireCount(), + "empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(), + "canceled_acquire_count": s.CanceledAcquireCount(), + } + })) +} + func newLogger(cfg config.Config) *slog.Logger { opts := &slog.HandlerOptions{Level: cfg.LogLevel} if cfg.Env == "dev" { diff --git a/compose.yaml b/compose.yaml index ccaa6ab9..5e1421eb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -165,6 +165,25 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M + # Load generator of the experiments (tools/loadgen). Only + # `docker compose --profile bench run --rm loadgen ...` starts it; it talks to + # the balancer and, for metrics, to the replicas' internal debug port. + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab3 + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-1.5}" + memory: 512M + volumes: pgdata: valkeydata: diff --git a/internal/server/server.go b/internal/server/server.go index 718c66df..c33fc2b1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -67,3 +67,24 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, log.Info("shutdown completed") return nil } + +// RunDebug serves h, the internal metrics of the process, on ln until ctx is +// cancelled. Nothing outside the stand's network reaches it and no request +// depends on it, so it closes at once instead of draining. +func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + go func() { + <-ctx.Done() + _ = srv.Close() + }() + log.Info("debug listening", "addr", ln.Addr().String()) + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 4f447f85..8a2caa39 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -143,3 +143,27 @@ func waitFor(t *testing.T, url string, status int) { } t.Fatalf("%s did not return %d in time", url, status) } + +// The debug listener has no drain: it serves the benchmark's metrics scraper +// only and stops with the process. +func TestRunDebugServesUntilCancelled(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.RunDebug(ctx, log, ln, h) }() + waitFor(t, "http://"+ln.Addr().String()+"/debug/vars", http.StatusTeapot) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunDebug returned %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunDebug did not stop after cancel") + } +} diff --git a/requirements-bench.txt b/requirements-bench.txt new file mode 100644 index 00000000..b1ec8530 --- /dev/null +++ b/requirements-bench.txt @@ -0,0 +1,2 @@ +# Plotting of the track 1 results (scripts/bench/plot.py); install into a venv. +matplotlib==3.10.3 diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh new file mode 100755 index 00000000..624ede52 --- /dev/null +++ b/scripts/bench/failover.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Level 4: one API replica fails under a constant write load. The generator +# sends messages at RATE for DURATION and retries uncertain sends with the +# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM, +# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started +# again. Afterwards the generator reads the history back: every confirmed +# message once. +# +# sh scripts/bench/failover.sh stop|kill +# +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), +# OUT (default evidence/failover). Results in /-/: +# timeline.json, seconds.csv (per-second outcomes, attributed by send time), +# faults.json (when and what was stopped), compose.log (the balancer's +# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. +# Runs on the current stand (`make up`) and leaves every replica running. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +kind=${1:?usage: failover.sh stop|kill } +n=${2:?usage: failover.sh stop|kill } +case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac +rate=${RATE:-50} +duration=${DURATION:-60s} +fault_at=${FAULT_AT:-15} +restore_at=${RESTORE_AT:-35} +dir=${OUT:-evidence/failover}/$kind-$n +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +target=$(docker compose ps -q api | sed -n "${n}p") +[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; } +alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }') +meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ + "fault_at_s=$fault_at" "restore_at_s=$restore_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +# The generator seeds two users first; its timeline starts after that. The +# fault times below are absolute, as is started_at in timeline.json. +sleep "$fault_at" +fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker "$kind" "$target" >/dev/null +fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +sleep $((restore_at - fault_at)) +restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker compose start api >/dev/null 2>&1 +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json" +docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh new file mode 100644 index 00000000..d7b9d48c --- /dev/null +++ b/scripts/bench/lib.sh @@ -0,0 +1,57 @@ +# Shared helpers of the experiment scripts; sourced, not run. + +# abspath prints the absolute path of an existing directory in the form the +# Docker CLI accepts for a bind mount (a Windows path under Git Bash). +abspath() { + (cd "$1" && (pwd -W 2>/dev/null || pwd)) +} + +# meta writes the header of one run: what ran, when, on which commit and +# host. Arguments: file, then key=value pairs added to the record. +meta() { + file=$1 + shift + { + printf '{\n' + printf ' "date_utc": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf ' "commit": "%s",\n' "$(git rev-parse --short HEAD)" + printf ' "branch": "%s",\n' "$(git branch --show-current)" + printf ' "docker": "%s",\n' "$(docker version --format '{{.Server.Version}}')" + printf ' "host_cpus": "%s",\n' "$(docker info --format '{{.NCPU}}')" + printf ' "host_memory_bytes": "%s",\n' "$(docker info --format '{{.MemTotal}}')" + for pair in "$@"; do + printf ' "%s": "%s",\n' "${pair%%=*}" "${pair#*=}" + done + printf ' "go": "%s"\n' "$(go env GOVERSION 2>/dev/null || echo unknown)" + printf '}\n' + } >"$file" +} + +# sample_docker_stats appends the CPU and memory of every container of the +# stand to a JSON-lines file, about every two seconds, until killed. +sample_docker_stats() { + while :; do + docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ + $(docker ps -q --filter label=com.docker.compose.project=lab3) 2>/dev/null || true + done >>"$1" +} + +# sample_postgres appends, every second, the database's commit counters and +# what its active sessions are waiting for (none: running on CPU). +sample_postgres() { + while :; do + docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " + SELECT json_build_object( + 't', extract(epoch FROM now()), + 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, + 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), + 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( + SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n + FROM pg_stat_activity + WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid() + GROUP BY 1) AS x)) + FROM pg_stat_database WHERE datname = 'messenger'" 2>/dev/null || true + sleep 1 + done >>"$1" +} diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py new file mode 100755 index 00000000..20493b2d --- /dev/null +++ b/scripts/bench/plot.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Tables and charts of track 1 from the raw results of run-series.sh. + + python scripts/bench/plot.py + +The raw root holds //rep/ directories. Written to the +output directory: + summary.csv one row per step of every run: offered and successful rate, + errors, latency percentiles, generator lag, mean CPU of each + component and pool waits during the step; + capacity.csv the capacity of every run and the median per configuration; + -throughput.png, -p99.png, capacity.png, + cpu-at-capacity.png, pool-waits.png. +Capacity is the highest valid step (generator on schedule) with less than 1 % +errors and a p99 of successful requests within 500 ms. +""" +import csv +import json +import pathlib +import statistics +import sys +from datetime import datetime + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +SLO_P99_MS = 500.0 +MAX_ERROR_SHARE = 0.01 +COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") + + +def read_jsonl(path): + if not path.exists(): + return [] + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass + return rows + + +def component(name): + for c in COMPONENTS: + if f"-{c}-" in name: + return c + return None + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def load_runs(root): + runs = [] + for steps_path in sorted(root.glob("*/*/rep*/steps.json")): + d = steps_path.parent + report = json.loads(steps_path.read_text(encoding="utf-8")) + runs.append({ + "series": d.parent.parent.name, + "config": d.parent.name, + "rep": d.name, + "start": parse_time(report["started_at"]), + "steps": report["steps"], + "docker": read_jsonl(d / "docker-stats.jsonl"), + "metrics": read_jsonl(d / "metrics.jsonl"), + }) + return runs + + +def mean_cpu(run, t0, t1): + """Mean CPU per component in [t0, t1); replicas of the API are summed.""" + per_time = {} + for row in run["docker"]: + c = component(row.get("name", "")) + if c is None or not t0 <= row["t"] < t1: + continue + try: + cpu = float(row["cpu"].rstrip("%")) + except (ValueError, KeyError): + continue + per_time.setdefault(row["t"], {}).setdefault(c, 0.0) + per_time[row["t"]][c] += cpu + out = {} + for c in COMPONENTS: + values = [v[c] for v in per_time.values() if c in v] + out[c] = round(statistics.mean(values), 1) if values else "" + return out + + +def pool_waits(run, t0, t1): + """Requests that waited for a pooled connection during [t0, t1), all + replicas together, per second.""" + first, last = {}, {} + for row in run["metrics"]: + t = run["start"] + row["t"] + if not t0 <= t < t1: + continue + for replica in row.get("replicas") or []: + pool = replica.get("pgxpool") or {} + key = replica.get("instance") + if key and "empty_acquire_count" in pool: + first.setdefault(key, pool["empty_acquire_count"]) + last[key] = pool["empty_acquire_count"] + if not last: + return "" + return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) + + +def is_capacity(step): + return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS + + +def main(root, out): + runs = load_runs(root) + if not runs: + sys.exit(f"no runs under {root}") + out.mkdir(parents=True, exist_ok=True) + + rows, capacities = [], [] + for run in runs: + best = None + for step in run["steps"]: + t0 = run["start"] + step["started_at_s"] + t1 = t0 + step["seconds"] + cpu = mean_cpu(run, t0, t1) + row = { + "series": run["series"], "config": run["config"], "rep": run["rep"], + "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), + "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], + "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], + **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, + "pool_waits_per_s": pool_waits(run, t0, t1), + } + rows.append(row) + if is_capacity(step): + best = row + capacities.append({ + "series": run["series"], "config": run["config"], "rep": run["rep"], + "capacity_rps": best["offered_rps"] if best else 0, "at": best, + }) + + with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + configs = [] + for c in capacities: + if (c["series"], c["config"]) not in configs: + configs.append((c["series"], c["config"])) + with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median"]) + medians = {} + for key in configs: + values = [c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == key] + medians[key] = statistics.median(values) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + + def median_by_offered(series, config, field): + points = {} + for r in rows: + if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "": + points.setdefault(r["offered_rps"], []).append(float(r[field])) + xs = sorted(points) + return xs, [statistics.median(points[x]) for x in xs] + + for series in sorted({s for s, _ in configs}): + names = [c for s, c in configs if s == series] + for field, title, filename, log in ( + ("ok_rps", "Successful requests per second", "throughput", False), + ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), + ): + fig, ax = plt.subplots(figsize=(7, 4.5)) + top = 0 + for name in names: + xs, ys = median_by_offered(series, name, field) + ax.plot(xs, ys, marker="o", label=name) + top = max([top, *xs]) + if field == "ok_rps" and top: + ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") + if field == "ok_p99_ms": + ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") + if log: + ax.set_yscale("log") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel(title) + ax.set_title(f"Series {series}: median of repeats, valid steps only") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / f"{series}-{filename}.png", dpi=120) + plt.close(fig) + + labels = [f"{s}/{c}" for s, c in configs] + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) + meds = [medians[k] for k in configs] + lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] + highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] + ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") + ax.set_ylabel("Capacity, requests per second") + ax.set_title("Capacity per configuration (median, range of repeats)") + ax.tick_params(axis="x", rotation=30) + ax.grid(axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(out / "capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) + width = 0.8 / len(COMPONENTS) + for i, comp in enumerate(COMPONENTS): + values = [] + for key in configs: + at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] + cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] + values.append(statistics.median(cpu) if cpu else 0) + ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) + ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) + ax.set_ylabel("Mean CPU at capacity, % of one core") + ax.set_title("Which component is busy at capacity") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cpu-at-capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(7, 4.5)) + for series, name in configs: + xs, ys = median_by_offered(series, name, "pool_waits_per_s") + if xs: + ax.plot(xs, ys, marker="o", label=f"{series}/{name}") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel("Waits for a pooled connection per second") + ax.set_title("Connection pool waits, all replicas") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "pool-waits.png", dpi=120) + plt.close(fig) + print(f"{len(runs)} runs, {len(rows)} steps -> {out}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])) diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh new file mode 100755 index 00000000..1bab9a47 --- /dev/null +++ b/scripts/bench/rollout-under-load.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Level 4: a rolling update under a constant write load. The generator sends +# messages at RATE for DURATION, retrying uncertain sends with the same key; +# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards +# the generator reads the history back: every confirmed message once. +# +# sh scripts/bench/rollout-under-load.sh +# +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default +# evidence/rollout). Results in /: timeline.json, seconds.csv, +# rollout.log, update.json (when it ran), compose.log, instances-*.txt. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +rate=${RATE:-50} +duration=${DURATION:-60s} +update_at=${UPDATE_AT:-15} +dir=${OUT:-evidence/rollout} +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +# Build first, as a delivery pipeline would: the update under load then +# measures the replacement of the replicas, not the compiler. +docker compose build api >/dev/null 2>&1 +meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$update_at" +started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +status=0 +sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? +finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \ + "$started" "$finished" "$status" >"$dir/update.json" +docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh new file mode 100755 index 00000000..df62c9e7 --- /dev/null +++ b/scripts/bench/run-series.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Track 1, the cost of scaling: runs every configuration of one series file, +# each repeated, each on a fresh stand (empty volumes, same seed), under the +# same grid of offered rates. +# +# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats] +# +# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root, +# default evidence/track1/raw). Fixed for every configuration: HAProxy +# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the +# generator LOADGEN_CPUS (1.5). Results per run: +# ///rep/{meta.json,steps.json,seconds.csv, +# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log} +# The stand is removed at the end; `make up` brings the normal one back. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +series=${1:?usage: run-series.sh [repeats]} +repeats=${2:-3} +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}" +step=${STEP:-30s} +warmup=${WARMUP:-30s} +root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf) + +export APP_DEBUG_ADDR=:9090 LB_QUIET=1 +export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5} +export MSYS_NO_PATHCONV=1 + +grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do + [ -n "$name" ] || continue + for rep in $(seq 1 "$repeats"); do + dir=$root/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool + + docker compose down -v --remove-orphans >/dev/null 2>&1 + docker compose up -d --build --wait >/dev/null 2>&1 + meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \ + "lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \ + "rates=$RATES" "step=$step" "warmup=$warmup" + + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log" + kill "$stats" "$pg" 2>/dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + cat "$dir/loadgen.log" | grep '^step' || true + done +done +docker compose down -v --remove-orphans >/dev/null 2>&1 +echo "results in $root" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf new file mode 100644 index 00000000..1a305380 --- /dev/null +++ b/scripts/bench/series/A.conf @@ -0,0 +1,6 @@ +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas. +# The pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +A1 1 2.0 1024M 920MiB 10 +A2 2 1.0 512M 460MiB 10 +A4 4 0.5 256M 230MiB 10 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf new file mode 100644 index 00000000..d0952eff --- /dev/null +++ b/scripts/bench/series/Aprime.conf @@ -0,0 +1,5 @@ +# Series A': the budget of series A with the total pool fixed at 20 +# connections. A'2 equals A2 and is not run again. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +Ap1 1 2.0 1024M 920MiB 20 +Ap4 4 0.5 256M 230MiB 5 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf new file mode 100644 index 00000000..b882d44e --- /dev/null +++ b/scripts/bench/series/B.conf @@ -0,0 +1,7 @@ +# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by +# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run +# again. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +B1 1 1.0 512M 460MiB 10 +B3 3 1.0 512M 460MiB 10 +B4 4 1.0 512M 460MiB 10 diff --git a/scripts/bench/series/calibrate.conf b/scripts/bench/series/calibrate.conf new file mode 100644 index 00000000..0e3c976e --- /dev/null +++ b/scripts/bench/series/calibrate.conf @@ -0,0 +1,4 @@ +# Calibration: the smallest API budget of the study (one replica, one CPU) +# with a wide range of rates finds where the common rate grid must reach. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +cal 1 1.0 512M 460MiB 10 diff --git a/scripts/restart.sh b/scripts/restart.sh index 11591f9c..29aeb757 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,6 +1,7 @@ #!/bin/sh # Restart hook of `make test-persistence` (tests/resilience.py): stops the -# API, PostgreSQL and Valkey gracefully and starts the same containers again. +# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the +# same containers again (the one-shot migrate finds nothing to apply). # Volumes and data are kept and nothing is re-seeded. The test hides the # output, so failures surface through the exit code only. set -eu diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go index 62bd2d5e..b56eb137 100644 --- a/tools/concurrency/client.go +++ b/tools/concurrency/client.go @@ -10,6 +10,8 @@ import ( "time" "github.com/google/uuid" + + "messenger/internal/chat" ) // client is a minimal typed HTTP client for the messenger API. @@ -50,6 +52,40 @@ func (c *client) postMessage(chatID, token, text string, out any) error { return c.do(req, token, http.StatusCreated, out) } +// sendWithKey sends one attempt of a message under the given key on a +// connection of its own and reports the status, the message (on 201) and the +// replica that answered. +func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { + var msg chat.MessageResponse + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return 0, msg, "", err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return 0, msg, "", err + } + req.Close = true + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Idempotency-Key", key) + resp, err := c.http.Do(req) + if err != nil { + return 0, msg, "", err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return 0, msg, "", err + } + if resp.StatusCode == http.StatusCreated { + if err := json.Unmarshal(payload, &msg); err != nil { + return 0, msg, "", err + } + } + return resp.StatusCode, msg, resp.Header.Get("X-Instance-Id"), nil +} + // login opens a session with Basic credentials and returns its bearer token. func (c *client) login(username, password string) (string, error) { req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go index eea5e108..7011c071 100644 --- a/tools/concurrency/main.go +++ b/tools/concurrency/main.go @@ -1,12 +1,22 @@ -// Command concurrency is the black-box race check from the grading rubric: -// N clients send M messages each into one chat at the same time, then the -// history is paged and must contain exactly N*M unique messages in id order. +// Command concurrency is the black-box race check from the grading rubric. +// +// Phase 1: N clients send M messages each into one chat at the same time, +// then the history is paged and must contain exactly N*M unique messages in +// id order. +// +// Phase 2, shared idempotency: in each of K rounds one user sends the same +// message with the same Idempotency-Key P times in parallel, each attempt on +// its own connection, so the balancer spreads them over the replicas. Every +// attempt must answer 201 with the same message, the attempts must have +// reached at least two replicas, the history must hold exactly K messages, +// and the key with another text must answer 409. package main import ( "encoding/json" "flag" "fmt" + "net/http" "os" "path/filepath" "sort" @@ -35,17 +45,37 @@ type Report struct { Ordered bool `json:"history_ordered_by_id"` Pages int `json:"pages_read"` ElapsedSeconds float64 `json:"elapsed_seconds"` - Passed bool `json:"passed"` + // Idempotency is phase 2. + Idempotency IdempotencyReport `json:"idempotency"` + Passed bool `json:"passed"` +} + +type IdempotencyReport struct { + Rounds int `json:"rounds"` + Parallel int `json:"parallel"` + // SameMessageRounds counts rounds whose attempts all answered 201 with + // one and the same message. + SameMessageRounds int `json:"same_message_rounds"` + // InstancesSeen is the number of distinct X-Instance-Id values among all + // attempts. + InstancesSeen int `json:"instances_seen"` + RequireReplicas bool `json:"require_two_replicas"` + Conflict409 bool `json:"different_body_409"` + History int `json:"history_messages"` + Passed bool `json:"passed"` } func main() { baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") clients := flag.Int("clients", 10, "number of concurrent clients") perClient := flag.Int("per-client", 10, "messages sent by each client") + rounds := flag.Int("rounds", 20, "idempotency rounds, one key each") + parallel := flag.Int("parallel", 8, "parallel attempts with the same key per round") + requireReplicas := flag.Bool("require-replicas", true, "fail unless the attempts reached at least two replicas") output := flag.String("output", "", "optional path for the JSON report") flag.Parse() - report, err := run(*baseURL, *clients, *perClient) + report, err := run(*baseURL, *clients, *perClient, *rounds, *parallel, *requireReplicas) if err != nil { fail(err) } @@ -61,6 +91,8 @@ func main() { } fmt.Printf("PASS: %d clients x %d messages produced exactly %d unique ordered messages\n", report.Clients, report.PerClient, report.Unique) + fmt.Printf("PASS: %d rounds x %d parallel attempts with one key produced %d messages across %d instances\n", + report.Idempotency.Rounds, report.Idempotency.Parallel, report.Idempotency.History, report.Idempotency.InstancesSeen) } func fail(err error) { @@ -75,9 +107,9 @@ func writeReport(path string, text []byte) error { return os.WriteFile(path, append(text, '\n'), 0o644) } -func run(baseURL string, clients, perClient int) (Report, error) { - if clients < 1 || perClient < 1 { - return Report{}, fmt.Errorf("clients and per-client must be positive") +func run(baseURL string, clients, perClient, rounds, parallel int, requireReplicas bool) (Report, error) { + if clients < 2 || perClient < 1 || rounds < 1 || parallel < 1 { + return Report{}, fmt.Errorf("clients must be at least 2; per-client, rounds and parallel positive") } report := Report{ BaseURL: baseURL, @@ -112,15 +144,82 @@ func run(baseURL string, clients, perClient int) (Report, error) { report.Listed = len(listed) report.Unique, report.Missing = compare(created, listed) report.Ordered = sort.StringsAreSorted(listed) + report.Idempotency, err = checkIdempotency(api, baseURL, runID, members[:2], tokens[0], rounds, parallel, requireReplicas) + if err != nil { + return report, err + } report.Passed = report.Failed == 0 && report.Created == report.Expected && report.Listed == report.Expected && report.Unique == report.Expected && report.Missing == 0 && - report.Ordered + report.Ordered && + report.Idempotency.Passed return report, nil } +// checkIdempotency runs phase 2 in a chat of its own. +func checkIdempotency(api *client, baseURL, runID string, members []string, token string, rounds, parallel int, requireReplicas bool) (IdempotencyReport, error) { + r := IdempotencyReport{Rounds: rounds, Parallel: parallel, RequireReplicas: requireReplicas} + room, err := createChat(api, runID+"-idem", members, token) + if err != nil { + return r, err + } + instances := map[string]bool{} + var key, text string + for round := range rounds { + key, text = uuid.NewString(), fmt.Sprintf("idempotency round %d", round) + type attempt struct { + status int + id string + instance string + } + attempts := make([]attempt, parallel) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range parallel { + wg.Go(func() { + own := newClient(baseURL) + <-start + status, msg, instance, err := own.sendWithKey(room.ID, token, key, text) + if err == nil { + attempts[i] = attempt{status, msg.ID, instance} + } + }) + } + close(start) + wg.Wait() + same := true + for _, a := range attempts { + if a.instance != "" { + instances[a.instance] = true + } + if a.status != http.StatusCreated || a.id == "" || a.id != attempts[0].id { + same = false + } + } + if same { + r.SameMessageRounds++ + } + } + r.InstancesSeen = len(instances) + status, _, _, err := newClient(baseURL).sendWithKey(room.ID, token, key, text+" changed") + if err != nil { + return r, err + } + r.Conflict409 = status == http.StatusConflict + listed, _, err := readHistory(api, room.ID, token) + if err != nil { + return r, err + } + r.History = len(listed) + r.Passed = r.SameMessageRounds == rounds && + r.History == rounds && + r.Conflict409 && + (!requireReplicas || r.InstancesSeen >= 2) + return r, nil +} + // clientPassword is shared by the synthetic clients of one run. const clientPassword = "concurrency-check-password" diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go index 7c9158b4..f0e1b045 100644 --- a/tools/concurrency/main_test.go +++ b/tools/concurrency/main_test.go @@ -43,7 +43,7 @@ func TestRunAgainstLiveServer(t *testing.T) { srv := httptest.NewServer(a.Engine) defer srv.Close() - report, err := run(srv.URL, 10, 10) + report, err := run(srv.URL, 10, 10, 5, 4, false) if err != nil { t.Fatalf("run error: %v", err) } @@ -56,10 +56,34 @@ func TestRunAgainstLiveServer(t *testing.T) { if !report.Ordered || !report.Passed { t.Fatalf("report = %+v", report) } + idem := report.Idempotency + if idem.Rounds != 5 || idem.Parallel != 4 || idem.SameMessageRounds != 5 || idem.History != 5 || !idem.Conflict409 || !idem.Passed { + t.Fatalf("idempotency = %+v", idem) + } + if idem.InstancesSeen != 1 { + t.Fatalf("one in-process server must be seen as one instance: %+v", idem) + } +} + +// Through the balancer the same key must reach at least two replicas; one +// replica alone does not prove the idempotency is shared. +func TestIdempotencyRequiresTwoReplicasByDefault(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 2, 1, 2, 2, true) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Idempotency.Passed || report.Passed { + t.Fatalf("a single instance passed the shared idempotency check: %+v", report.Idempotency) + } } func TestRunFailsOnUnreachableServer(t *testing.T) { - if _, err := run("http://127.0.0.1:1", 2, 2); err == nil { + if _, err := run("http://127.0.0.1:1", 2, 2, 1, 1, false); err == nil { t.Fatal("expected an error for an unreachable server") } } diff --git a/tools/loadgen/client.go b/tools/loadgen/client.go new file mode 100644 index 00000000..fce2c380 --- /dev/null +++ b/tools/loadgen/client.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// result is one HTTP exchange; instance is the X-Instance-Id of the answer, +// empty when the balancer answered itself or nothing answered. +type result struct { + status int + instance string + body []byte + latency time.Duration + err error +} + +type client struct { + base string + http *http.Client + timeout time.Duration +} + +// newClient keeps many connections open to the balancer, as many concurrent +// users would; HAProxy balances every request, not every connection. +func newClient(base string, timeout time.Duration) *client { + transport := &http.Transport{ + MaxIdleConns: 10000, + MaxIdleConnsPerHost: 10000, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + } + return &client{base: base, http: &http.Client{Transport: transport}, timeout: timeout} +} + +func (c *client) do(ctx context.Context, method, path, token, key string, body []byte, basic string) result { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return result{err: err} + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + switch { + case basic != "": + req.Header.Set("Authorization", "Basic "+basic) + case token != "": + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + start := time.Now() + resp, err := c.http.Do(req) + if err != nil { + return result{err: err, latency: time.Since(start)} + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + r := result{status: resp.StatusCode, instance: resp.Header.Get("X-Instance-Id"), body: payload, latency: time.Since(start), err: err} + return r +} + +func (c *client) postJSON(ctx context.Context, path, token, key string, body any) result { + raw, err := json.Marshal(body) + if err != nil { + return result{err: err} + } + return c.do(ctx, http.MethodPost, path, token, key, raw, "") +} + +// expect decodes the body of a result with the wanted status. +func expect(r result, want int, out any) error { + if r.err != nil { + return r.err + } + if r.status != want { + return fmt.Errorf("status %d, want %d: %s", r.status, want, r.body) + } + if out == nil { + return nil + } + return json.Unmarshal(r.body, out) +} + +func (c *client) register(ctx context.Context, username, password string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]string{"username": username, "display_name": username, "password": password} + err := expect(c.postJSON(ctx, "/api/v1/users", "", "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) login(ctx context.Context, username, password string) (string, error) { + var session struct { + Token string `json:"token"` + } + basic := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + err := expect(c.do(ctx, http.MethodPost, "/api/v1/auth/sessions", "", "", nil, basic), http.StatusCreated, &session) + return session.Token, err +} + +func (c *client) createChat(ctx context.Context, token, title string, members []string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]any{"title": title, "member_ids": members} + err := expect(c.postJSON(ctx, "/api/v1/chats", token, "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) send(ctx context.Context, chatID, token, key, text string) result { + return c.postJSON(ctx, "/api/v1/chats/"+chatID+"/messages", token, key, map[string]string{"text": text}) +} + +// firstPage reads the oldest page of a history: the read of the profile. +func (c *client) firstPage(ctx context.Context, chatID, token string) result { + return c.do(ctx, http.MethodGet, "/api/v1/chats/"+chatID+"/messages?limit=50", token, "", nil, "") +} + +type message struct { + ID string `json:"id"` + Text string `json:"text"` +} + +// history reads a whole chat, page by page. +func (c *client) history(ctx context.Context, chatID, token string) ([]message, error) { + var all []message + path := "/api/v1/chats/" + chatID + "/messages?limit=100" + for range 100000 { + var page struct { + Items []message `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if err := expect(c.do(ctx, http.MethodGet, path, token, "", nil, ""), http.StatusOK, &page); err != nil { + return nil, err + } + all = append(all, page.Items...) + if page.NextCursor == nil { + return all, nil + } + path = "/api/v1/chats/" + chatID + "/messages?limit=100&cursor=" + url.QueryEscape(*page.NextCursor) + } + return nil, fmt.Errorf("history of %s did not end", chatID) +} diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go new file mode 100644 index 00000000..fabeddd7 --- /dev/null +++ b/tools/loadgen/main.go @@ -0,0 +1,352 @@ +// Command loadgen is the open-loop load generator of the scaling study +// (track 1) and of the failover and rolling-update experiments. +// +// loadgen bench — seeds users, chats and messages, warms up, then offers +// each rate of -rates for -step: a mix of history reads +// (-read-ratio) and message sends. Writes steps.json and +// seconds.csv, and metrics.jsonl with -scrape. +// loadgen timeline — one user sends messages into one chat at -rate for +// -duration; a send with an uncertain outcome (no answer, +// 5xx) is retried with the same Idempotency-Key. At the +// end the whole history is read back: every confirmed +// message must be there, and none twice. Writes +// timeline.json and seconds.csv. +// +// Requests are sent on schedule whatever the response times (open loop), and +// the generator records how late it sent them: a step it could not keep up +// with is marked invalid rather than reported as the system's limit. Tokens +// stay in memory and are never written out. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + mrand "math/rand/v2" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" +) + +type config struct { + Mode string `json:"mode"` + BaseURL string `json:"base_url"` + Users int `json:"users"` + Chats int `json:"chats"` + SeedMessages int `json:"seed_messages_per_chat"` + ReadRatio float64 `json:"read_ratio"` + Rates []float64 `json:"rates"` + Step string `json:"step"` + Warmup string `json:"warmup"` + Rate float64 `json:"rate"` + Duration string `json:"duration"` + Timeout string `json:"client_timeout"` + Seed uint64 `json:"seed"` + Scrape bool `json:"scrape"` + StatsURL string `json:"stats_url"` + APIHost string `json:"api_host"` + DebugPort string `json:"debug_port"` + Out string `json:"-"` + + step, warmup, duration, timeout time.Duration +} + +func main() { + if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") { + fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags") + os.Exit(2) + } + cfg, err := parse(os.Args[1], os.Args[2:]) + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if cfg.Mode == "bench" { + err = runBench(ctx, cfg) + } else { + err = runTimeline(ctx, cfg) + } + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(1) + } +} + +func parse(mode string, args []string) (config, error) { + cfg := config{Mode: mode} + fs := flag.NewFlagSet(mode, flag.ContinueOnError) + rates := fs.String("rates", "200,400", "bench: offered requests per second of each step") + fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API") + fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users") + fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members") + fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring") + fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends") + fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step") + fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate") + fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second") + fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") + fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") + fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") + fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") + fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") + fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)") + fs.StringVar(&cfg.Out, "out", "/out", "directory for the results") + if err := fs.Parse(args); err != nil { + return cfg, err + } + for _, part := range strings.Split(*rates, ",") { + r, err := strconv.ParseFloat(strings.TrimSpace(part), 64) + if err != nil || r <= 0 { + return cfg, fmt.Errorf("rates: %q is not a positive number", part) + } + cfg.Rates = append(cfg.Rates, r) + } + for _, d := range []struct { + name string + raw string + dst *time.Duration + }{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} { + v, err := time.ParseDuration(d.raw) + if err != nil || v < 0 { + return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw) + } + *d.dst = v + } + if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 { + return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive") + } + return cfg, os.MkdirAll(cfg.Out, 0o755) +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0o644) +} + +// startScrape samples the system in the background when asked to; the +// returned function stops it and waits. +func startScrape(ctx context.Context, cfg config, start time.Time) func() { + if !cfg.Scrape { + return func() {} + } + ctx, cancel := context.WithCancel(ctx) + s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}} + var wg sync.WaitGroup + wg.Go(func() { + if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil { + fmt.Fprintln(os.Stderr, "loadgen: scrape:", err) + } + }) + return func() { cancel(); wg.Wait() } +} + +type benchReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Users int `json:"users"` + Chats int `json:"chats"` + Seeded int `json:"seeded_messages"` + Steps []stepSummary `json:"steps"` + WritesOK int `json:"messages_written_during_steps"` +} + +func runBench(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) + ds, err := seed(ctx, c, rng, cfg.Users, cfg.Chats, cfg.SeedMessages) + if err != nil { + return err + } + report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages} + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + + // next picks the request on the scheduling goroutine, where the random + // source may be used safely. + counter := 0 + next := func(rec *recorder) func(time.Time) func() { + return func(planned time.Time) func() { + r := ds.rooms[rng.IntN(len(ds.rooms))] + a := ds.actors[r.members[rng.IntN(len(r.members))]] + write := rng.Float64() >= cfg.ReadRatio + counter++ + text := fmt.Sprintf("load %d", counter) + return func() { + sent := time.Now() + var res result + if write { + res = c.send(ctx, r.id, a.token, uuid.NewString(), text) + } else { + res = c.firstPage(ctx, r.id, a.token) + } + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write}) + } + } + } + if cfg.warmup > 0 { + openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{})) + } + all := &recorder{} + for _, rate := range cfg.Rates { + if ctx.Err() != nil { + break + } + stepStart := time.Now() + rec := &recorder{} + openLoop(ctx, rate, cfg.step, next(rec)) + s := summarize(rate, cfg.step, rec.all()) + s.StartedAt = stepStart.Sub(start).Seconds() + report.Steps = append(report.Steps, s) + report.WritesOK += s.Writes + for _, x := range rec.all() { + all.add(x) + } + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid) + } + stopScrape() + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { + return err + } + return writeJSON(filepath.Join(cfg.Out, "steps.json"), report) +} + +type timelineReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Jobs int `json:"messages_attempted"` + Attempts int `json:"requests"` + Confirmed int `json:"confirmed"` + RetriedJobs int `json:"messages_retried"` + Unresolved int `json:"unresolved"` + Rejected int `json:"rejected_4xx"` + Classes map[string]int `json:"classes"` + Instances map[string]int `json:"instances"` + History int `json:"history_messages"` + Duplicates int `json:"duplicates"` + Missing int `json:"missing_confirmed"` + Passed bool `json:"passed"` +} + +// retryDelay spaces the attempts of one message whose outcome was uncertain. +const retryDelay = 200 * time.Millisecond + +// retryGrace is how long after the schedule ends an uncertain message keeps +// being retried before it counts as unresolved. +const retryGrace = 10 * time.Second + +func runTimeline(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) + ds, err := seed(ctx, c, rng, 2, 1, 0) + if err != nil { + return err + } + r := ds.rooms[0] + sender := ds.actors[r.members[0]] + report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}} + runID := randomHex(3) + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + deadline := start.Add(cfg.duration + retryGrace) + + var mu sync.Mutex + confirmed := map[string]string{} // text -> message id + rec := &recorder{} + jobs := 0 + openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() { + jobs++ + key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs) + return func() { + for attempt := 0; ; attempt++ { + sent := time.Now() + res := c.send(ctx, r.id, sender.token, key, text) + cls := classify(res) + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true}) + mu.Lock() + switch { + case cls == "ok": + var m message + if json.Unmarshal(res.body, &m) == nil { + confirmed[text] = m.ID + } + case res.err == nil && res.status >= 400 && res.status < 500: + report.Rejected++ + case time.Now().After(deadline) || ctx.Err() != nil: + report.Unresolved++ + default: + if attempt == 0 { + report.RetriedJobs++ + } + mu.Unlock() + time.Sleep(retryDelay) + planned = time.Now() + continue + } + mu.Unlock() + return + } + } + }) + stopScrape() + samples := rec.all() + report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed) + for _, x := range samples { + report.Classes[x.class]++ + if x.instance != "" { + report.Instances[x.instance]++ + } + } + + history, err := c.history(ctx, r.id, sender.token) + if err != nil { + return fmt.Errorf("read history back: %w", err) + } + report.History = len(history) + seen := map[string]int{} + byID := map[string]string{} + for _, m := range history { + seen[m.Text]++ + byID[m.ID] = m.Text + } + for _, n := range seen { + if n > 1 { + report.Duplicates += n - 1 + } + } + for text, id := range confirmed { + if byID[id] != text { + report.Missing++ + } + } + report.Passed = report.Duplicates == 0 && report.Missing == 0 + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil { + return err + } + if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil { + return err + } + fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n", + report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing) + if !report.Passed { + return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing) + } + return nil +} diff --git a/tools/loadgen/main_test.go b/tools/loadgen/main_test.go new file mode 100644 index 00000000..78dba77c --- /dev/null +++ b/tools/loadgen/main_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func liveServer(t *testing.T) string { + t.Helper() + a := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "loadgen-test", + }) + srv := httptest.NewServer(a.Engine) + t.Cleanup(srv.Close) + return srv.URL +} + +func TestBenchAgainstLiveServer(t *testing.T) { + out := t.TempDir() + cfg, err := parse("bench", []string{"-base-url", liveServer(t), "-users", "4", "-chats", "2", "-seed-messages", "3", + "-rates", "20,40", "-step", "1s", "-warmup", "0s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report benchReport + raw, err := os.ReadFile(filepath.Join(out, "steps.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("steps.json: %v %s", err, raw) + } + if len(report.Steps) != 2 || report.Seeded != 6 || report.Users != 4 || report.Chats != 2 { + t.Fatalf("report = %+v", report) + } + for _, s := range report.Steps { + if s.Sent != int(s.OfferedRPS) || s.OK != s.Sent || s.ErrorShare != 0 || s.Instances["loadgen-test"] != s.Sent { + t.Fatalf("step = %+v", s) + } + } + csv, err := os.ReadFile(filepath.Join(out, "seconds.csv")) + if err != nil || !strings.HasPrefix(string(csv), "second,sent,ok,") || strings.Count(string(csv), "\n") < 3 { + t.Fatalf("seconds.csv: %v\n%s", err, csv) + } +} + +func TestTimelineVerifiesHistory(t *testing.T) { + out := t.TempDir() + cfg, err := parse("timeline", []string{"-base-url", liveServer(t), "-rate", "20", "-duration", "1s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runTimeline(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report timelineReport + raw, err := os.ReadFile(filepath.Join(out, "timeline.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("timeline.json: %v %s", err, raw) + } + if report.Jobs != 20 || report.Confirmed != 20 || report.History != 20 || report.Duplicates != 0 || report.Missing != 0 || !report.Passed { + t.Fatalf("report = %+v", report) + } +} diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go new file mode 100644 index 00000000..21019fd3 --- /dev/null +++ b/tools/loadgen/run.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "sync" + "time" +) + +// openLoop offers requests at a constant rate for d, whatever the response +// times: request i is planned at start + i/rate and sent then, or at once if +// the generator is already late. Waiting for responses before sending more +// (a closed loop) would let a slow system lower its own load and hide its +// latency: coordinated omission. next runs on the scheduling goroutine, so it +// may use non-thread-safe state such as the random source; the job it +// returns runs on a goroutine of its own. openLoop returns once every job +// has finished. +func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { + n := int(rate * d.Seconds()) + interval := time.Duration(float64(time.Second) / rate) + start := time.Now() + var wg sync.WaitGroup + for i := range n { + planned := start.Add(time.Duration(i) * interval) + if wait := time.Until(planned); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + if ctx.Err() != nil { + break + } + wg.Go(next(planned)) + } + wg.Wait() +} + +// recorder collects samples from concurrent jobs. +type recorder struct { + mu sync.Mutex + samples []sample +} + +func (r *recorder) add(s sample) { + r.mu.Lock() + r.samples = append(r.samples, s) + r.mu.Unlock() +} + +func (r *recorder) all() []sample { + r.mu.Lock() + defer r.mu.Unlock() + return append([]sample(nil), r.samples...) +} diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go new file mode 100644 index 00000000..69790f23 --- /dev/null +++ b/tools/loadgen/scrape.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "io" + "net" + "net/http" + "os" + "strings" + "time" +) + +// scraper samples, once a second, what the balancer and every replica report +// about themselves, so a saturated step can be explained: the HAProxy +// statistics per replica and the expvar of each API process (pool waits, +// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them, +// through Docker DNS, and asked directly on their internal debug port. +type scraper struct { + statsURL string + apiHost string + debugPort string + http *http.Client +} + +type scrapeLine struct { + T float64 `json:"t"` + HAProxy []map[string]string `json:"haproxy,omitempty"` + Replicas []json.RawMessage `json:"replicas,omitempty"` +} + +// haproxyFields are the columns of the statistics CSV that are kept. +var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"} + +func (s scraper) run(ctx context.Context, start time.Time, path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)} + if err := enc.Encode(line); err != nil { + return err + } + } +} + +func (s scraper) get(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := s.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) +} + +// haproxy keeps the rows of the replicas backend. +func (s scraper) haproxy(ctx context.Context) []map[string]string { + raw, err := s.get(ctx, s.statsURL) + if err != nil { + return nil + } + rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll() + if err != nil || len(rows) < 2 { + return nil + } + header := rows[0] + col := map[string]int{} + for i, name := range header { + col[name] = i + } + var out []map[string]string + for _, row := range rows[1:] { + if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") { + continue + } + kept := map[string]string{} + for _, name := range haproxyFields { + if i, ok := col[name]; ok { + kept[name] = row[i] + } + } + out = append(out, kept) + } + return out +} + +// replicas returns the expvar documents of the running replicas, trimmed to +// the fields the study uses. +func (s scraper) replicas(ctx context.Context) []json.RawMessage { + addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost) + if err != nil { + return nil + } + var out []json.RawMessage + for _, addr := range addrs { + raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars") + if err != nil { + continue + } + var vars struct { + Instance string `json:"instance"` + GOMAXPROCS int `json:"gomaxprocs"` + Pool json.RawMessage `json:"pgxpool"` + Mem struct { + HeapAlloc uint64 `json:"HeapAlloc"` + Sys uint64 `json:"Sys"` + NumGC uint32 `json:"NumGC"` + PauseTotalNs uint64 `json:"PauseTotalNs"` + GCCPUFraction float64 `json:"GCCPUFraction"` + } `json:"memstats"` + } + if json.Unmarshal(raw, &vars) != nil { + continue + } + kept, err := json.Marshal(map[string]any{ + "addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS, + "pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys, + "num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction, + }) + if err == nil { + out = append(out, kept) + } + } + return out +} diff --git a/tools/loadgen/seed.go b/tools/loadgen/seed.go new file mode 100644 index 00000000..748b0c18 --- /dev/null +++ b/tools/loadgen/seed.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + mrand "math/rand/v2" + "sync" + + "github.com/google/uuid" +) + +// seedPassword is shared by the synthetic users of one run. +const seedPassword = "loadgen-synthetic-password" + +// seedWorkers bounds the parallel registrations: each one hashes a password +// with Argon2id, and the setup must not be what the study measures. +const seedWorkers = 8 + +type actor struct { + id, token string +} + +type room struct { + id string + members []int // indexes into dataset.actors +} + +type dataset struct { + actors []actor + rooms []room + // messages is the number of messages written while seeding. + messages int +} + +// seed registers users and logs each in once, creates chats of two to five +// random members and writes perChat messages into each. Logins happen here, +// so the password hashing never shows up in the measured steps. +func seed(ctx context.Context, c *client, rng *mrand.Rand, users, chats, perChat int) (dataset, error) { + if users < 2 || chats < 1 { + return dataset{}, fmt.Errorf("need at least 2 users and 1 chat") + } + runID := randomHex(3) + ds := dataset{actors: make([]actor, users)} + if err := parallel(users, func(i int) error { + name := fmt.Sprintf("lg_%s_%d", runID, i) + id, err := c.register(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("register %s: %w", name, err) + } + token, err := c.login(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("log in %s: %w", name, err) + } + ds.actors[i] = actor{id: id, token: token} + return nil + }); err != nil { + return dataset{}, err + } + + for j := range chats { + size := 2 + rng.IntN(4) + members := rng.Perm(users)[:min(size, users)] + ids := make([]string, len(members)) + for k, m := range members { + ids[k] = ds.actors[m].id + } + id, err := c.createChat(ctx, ds.actors[members[0]].token, fmt.Sprintf("load %s %d", runID, j), ids) + if err != nil { + return dataset{}, fmt.Errorf("create chat %d: %w", j, err) + } + ds.rooms = append(ds.rooms, room{id: id, members: members}) + } + + if err := parallel(len(ds.rooms), func(j int) error { + r := ds.rooms[j] + for k := range perChat { + sender := ds.actors[r.members[k%len(r.members)]] + res := c.send(ctx, r.id, sender.token, uuid.NewString(), fmt.Sprintf("seed %d", k)) + if err := expect(res, 201, nil); err != nil { + return fmt.Errorf("seed message into chat %d: %w", j, err) + } + } + return nil + }); err != nil { + return dataset{}, err + } + ds.messages = len(ds.rooms) * perChat + return ds, nil +} + +// parallel runs fn(0..n-1) on seedWorkers goroutines and returns the first +// error. +func parallel(n int, fn func(i int) error) error { + var ( + wg sync.WaitGroup + mu sync.Mutex + first error + ) + next := make(chan int) + for range seedWorkers { + wg.Go(func() { + for i := range next { + if err := fn(i); err != nil { + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + } + } + }) + } + for i := range n { + next <- i + } + close(next) + wg.Wait() + return first +} + +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go new file mode 100644 index 00000000..f74336e9 --- /dev/null +++ b/tools/loadgen/stats.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "slices" + "sort" + "strings" + "time" +) + +// maxValidLag is the generator's own tolerance: if requests left more than +// this late at the 99th percentile, the generator, not the system, limited +// the step, and the step is excluded from the conclusions. +const maxValidLag = 10 * time.Millisecond + +// sample is one request as the generator saw it. +type sample struct { + planned time.Time + lag time.Duration // actual send time minus planned send time + latency time.Duration + class string + instance string + write bool +} + +// classify names the outcome of a request. HTTP errors produced by the +// balancer itself carry no X-Instance-Id and are kept apart from the API's +// own: a 503 from HAProxy means no replica was available, a 503 from the API +// means a dependency was. client_timeout is the generator giving up on a +// response, not a timeout of the system. +func classify(r result) string { + switch { + case r.err != nil && isTimeout(r.err): + return "client_timeout" + case r.err != nil: + return "conn_error" + case r.status >= 200 && r.status < 300: + return "ok" + case r.instance == "": + return fmt.Sprintf("http_%d_lb", r.status) + default: + return fmt.Sprintf("http_%d", r.status) + } +} + +func isTimeout(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// percentile is the nearest-rank percentile of an ascending slice. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + rank := int(math.Ceil(p / 100 * float64(len(sorted)))) + return sorted[min(max(rank, 1), len(sorted))-1] +} + +type stepSummary struct { + OfferedRPS float64 `json:"offered_rps"` + Seconds float64 `json:"seconds"` + StartedAt float64 `json:"started_at_s"` + Sent int `json:"sent"` + OK int `json:"ok"` + OKRPS float64 `json:"ok_rps"` + ErrorShare float64 `json:"error_share"` + Classes map[string]int `json:"classes"` + P50Ms float64 `json:"ok_p50_ms"` + P95Ms float64 `json:"ok_p95_ms"` + P99Ms float64 `json:"ok_p99_ms"` + LagP99Ms float64 `json:"lag_p99_ms"` + Valid bool `json:"valid"` + Instances map[string]int `json:"instances"` + Writes int `json:"writes_ok"` +} + +// summarize reduces one step. Latency percentiles cover successful requests +// only, as the course's report format asks; the errors are counted by class. +func summarize(rate float64, d time.Duration, samples []sample) stepSummary { + s := stepSummary{ + OfferedRPS: rate, + Seconds: d.Seconds(), + Sent: len(samples), + Classes: map[string]int{}, + Instances: map[string]int{}, + } + var ok, lags []time.Duration + for _, x := range samples { + s.Classes[x.class]++ + lags = append(lags, x.lag) + if x.instance != "" { + s.Instances[x.instance]++ + } + if x.class == "ok" { + ok = append(ok, x.latency) + if x.write { + s.Writes++ + } + } + } + slices.Sort(ok) + slices.Sort(lags) + s.OK = len(ok) + s.OKRPS = float64(s.OK) / d.Seconds() + if s.Sent > 0 { + s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent) + } + s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)) + lagP99 := percentile(lags, 99) + s.LagP99Ms = ms(lagP99) + s.Valid = lagP99 <= maxValidLag + return s +} + +func ms(d time.Duration) float64 { + return float64(d.Microseconds()) / 1000 +} + +// errorColumns are the fixed error columns of the per-second CSV; any other +// class lands in err_other. +var errorColumns = []string{"client_timeout", "conn_error"} + +// secondsCSV renders the per-second timeline, attributed by planned send +// time so it lines up with the moment a fault was injected. +func secondsCSV(start time.Time, samples []sample) string { + type bucket struct { + samples []sample + } + buckets := map[int]*bucket{} + last := 0 + for _, x := range samples { + sec := int(x.planned.Sub(start) / time.Second) + if buckets[sec] == nil { + buckets[sec] = &bucket{} + } + buckets[sec].samples = append(buckets[sec].samples, x) + last = max(last, sec) + } + var b strings.Builder + b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n") + for sec := 0; sec <= last; sec++ { + bk := buckets[sec] + if bk == nil { + fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec) + continue + } + var ok, lags []time.Duration + counts := map[string]int{} + instances := map[string]int{} + for _, x := range bk.samples { + lags = append(lags, x.lag) + if x.instance != "" { + instances[x.instance]++ + } + switch { + case x.class == "ok": + ok = append(ok, x.latency) + case slices.Contains(errorColumns, x.class): + counts[x.class]++ + case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"): + counts["http_5xx_lb"]++ + case strings.HasPrefix(x.class, "http_5"): + counts["http_5xx_api"]++ + case strings.HasPrefix(x.class, "http_4"): + counts["http_4xx"]++ + default: + counts["err_other"]++ + } + } + slices.Sort(ok) + slices.Sort(lags) + names := make([]string, 0, len(instances)) + for name := range instances { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, name := range names { + parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name])) + } + fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n", + sec, len(bk.samples), len(ok), + counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"], + ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)), + strings.Join(parts, "|")) + } + return b.String() +} diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go new file mode 100644 index 00000000..d9b32e1d --- /dev/null +++ b/tools/loadgen/stats_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" +) + +type timeoutErr struct{} + +func (timeoutErr) Error() string { return "i/o timeout" } +func (timeoutErr) Timeout() bool { return true } +func (timeoutErr) Temporary() bool { return true } + +var _ net.Error = timeoutErr{} + +func TestClassify(t *testing.T) { + for _, c := range []struct { + res result + want string + }{ + {result{status: 201, instance: "a"}, "ok"}, + {result{status: 200, instance: "a"}, "ok"}, + {result{status: 409, instance: "a"}, "http_409"}, + {result{status: 503, instance: "a"}, "http_503"}, + {result{status: 503}, "http_503_lb"}, + {result{status: 502}, "http_502_lb"}, + {result{err: context.DeadlineExceeded}, "client_timeout"}, + {result{err: fmt.Errorf("post: %w", timeoutErr{})}, "client_timeout"}, + {result{err: errors.New("connection refused")}, "conn_error"}, + } { + if got := classify(c.res); got != c.want { + t.Errorf("classify(%+v) = %q, want %q", c.res, got, c.want) + } + } +} + +func TestPercentileNearestRank(t *testing.T) { + ms := func(v ...int) []time.Duration { + out := make([]time.Duration, len(v)) + for i, x := range v { + out[i] = time.Duration(x) * time.Millisecond + } + return out + } + sorted := ms(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + for p, want := range map[float64]time.Duration{50: 5 * time.Millisecond, 95: 10 * time.Millisecond, 99: 10 * time.Millisecond, 10: time.Millisecond} { + if got := percentile(sorted, p); got != want { + t.Errorf("p%v = %v, want %v", p, got, want) + } + } + if got := percentile(nil, 99); got != 0 { + t.Errorf("empty p99 = %v, want 0", got) + } +} + +// A step is invalid when the generator itself fell behind its schedule: the +// system was then offered less than the nominal rate. +func TestStepSummaryValidityAndErrorShare(t *testing.T) { + samples := []sample{ + {class: "ok", latency: 10 * time.Millisecond, lag: time.Millisecond}, + {class: "ok", latency: 20 * time.Millisecond, lag: time.Millisecond}, + {class: "client_timeout", latency: 2 * time.Second, lag: time.Millisecond}, + {class: "http_503_lb", latency: time.Millisecond, lag: time.Millisecond}, + } + s := summarize(4, time.Second, samples) + if s.Sent != 4 || s.OK != 2 || s.OKRPS != 2 || s.ErrorShare != 0.5 || !s.Valid { + t.Fatalf("summary = %+v", s) + } + if s.Classes["client_timeout"] != 1 || s.Classes["http_503_lb"] != 1 { + t.Fatalf("classes = %v", s.Classes) + } + if s.P99Ms != 20 { + t.Fatalf("p99 of successful requests = %v ms, want 20", s.P99Ms) + } + samples[0].lag = 50 * time.Millisecond + samples[1].lag = 50 * time.Millisecond + if s := summarize(4, time.Second, samples); s.Valid { + t.Fatalf("lagging generator marked valid: %+v", s) + } +} + +// Open loop: requests leave on schedule even when responses are slow. A +// closed loop with one worker would send only a handful here. +func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { + var fired atomic.Int32 + start := time.Now() + openLoop(context.Background(), 50, time.Second, func(time.Time) func() { + return func() { + fired.Add(1) + time.Sleep(200 * time.Millisecond) + } + }) + if n := fired.Load(); n < 45 || n > 50 { + t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) + } + if took := time.Since(start); took > 1500*time.Millisecond { + t.Fatalf("open loop took %v, want about 1.2 s", took) + } +} -- 2.54.0 From c9d0f732212e4011134a1d8eeea614c73d153c1a Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 18:23:26 +0300 Subject: [PATCH 06/21] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E:=20=D0=B7=D0=B0?= =?UTF-8?q?=D1=89=D0=B8=D1=82=D0=B0=20rollout=20=D0=BE=D1=82=20=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D1=85=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA,=20=D1=87?= =?UTF-8?q?=D1=82=D0=B5=D0=BD=D0=B8=D0=B5=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9?= =?UTF-8?q?,=20=D0=B2=D1=80=D0=B5=D0=BC=D1=8F=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2=20=D0=BF?= =?UTF-8?q?=D0=BE=20=D1=87=D0=B0=D1=81=D0=B0=D0=BC=20Docker,=20=D0=B2?= =?UTF-8?q?=D1=8B=D1=85=D0=BE=D0=B4=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=B0=20=D0=BF=D1=80=D0=B8=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA?= =?UTF-8?q?=D0=B5=20=D1=81=D1=82=D0=B0=D1=80=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/server/main.go | 19 +++++--- cmd/server/main_test.go | 72 +++++++++++++++++++++++++++++ internal/auth/service_test.go | 18 ++++++-- scripts/bench/failover.sh | 23 ++++++--- scripts/bench/rollout-under-load.sh | 27 ++++++++--- scripts/bench/run-series.sh | 5 +- scripts/rollout.sh | 10 ++++ tools/loadgen/stats_test.go | 4 +- 8 files changed, 153 insertions(+), 25 deletions(-) create mode 100644 cmd/server/main_test.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 72e2440f..6bf7d66b 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -80,15 +80,22 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, }) - var background sync.WaitGroup - defer background.Wait() // before the deferred pool.Close - background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + var debugLn net.Listener if cfg.DebugAddr != "" { - publishDebugVars(alias, pool) - debugLn, err := net.Listen("tcp", cfg.DebugAddr) - if err != nil { + if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil { return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) } + publishDebugVars(alias, pool) + } + + // The background work ends with the server: on a signal, and also when + // the server fails, so the deferred Wait cannot block the exit. + var background sync.WaitGroup + defer background.Wait() // before the deferred pool.Close + ctx, cancel := context.WithCancel(ctx) + defer cancel() // before Wait + background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if debugLn != nil { background.Go(func() { if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { log.Error("debug listener stopped", "error", err.Error()) diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go new file mode 100644 index 00000000..3885495a --- /dev/null +++ b/cmd/server/main_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "os" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +// A start-up failure after the background work began must still end the +// process with an error, not leave it waiting for a signal. +func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) { + busy, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer busy.Close() + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + ShutdownTimeout: time.Second, + MaxBodyBytes: 1024, + DatabaseURL: db.DSN, + SessionStoreURL: kv.URL, + SessionTTL: time.Hour, + IdempotencyTTL: time.Hour, + DBMaxConns: 2, + DebugAddr: busy.Addr().String(), + } + done := make(chan error, 1) + go func() { + done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias") + }() + select { + case err := <-done: + if err == nil { + t.Fatal("run succeeded on a busy debug address") + } + case <-time.After(10 * time.Second): + t.Fatal("run hung instead of returning the listen error") + } +} diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go index fb5d767b..b5c8173f 100644 --- a/internal/auth/service_test.go +++ b/internal/auth/service_test.go @@ -129,7 +129,11 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) { } } +// ExpiresAt is written by this process's clock and enforced by Valkey's. In +// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host +// by a fraction of a second, so the expiry is awaited with a tolerance. func TestSessionExpiresAfterTTL(t *testing.T) { + const clockTolerance = 3 * time.Second f := newFixture(t, time.Second) issued, err := f.svc.Login(f.ctx, "alice", pw) if err != nil { @@ -138,8 +142,16 @@ func TestSessionExpiresAfterTTL(t *testing.T) { if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil { t.Fatalf("fresh session: %v", err) } - time.Sleep(time.Until(issued.ExpiresAt) + 100*time.Millisecond) - if _, err := f.svc.Authenticate(f.ctx, issued.Token); !errors.Is(err, auth.ErrUnauthenticated) { - t.Fatalf("expired session: err = %v, want ErrUnauthenticated", err) + time.Sleep(time.Until(issued.ExpiresAt)) + deadline := time.Now().Add(clockTolerance) + for { + _, err := f.svc.Authenticate(f.ctx, issued.Token) + if errors.Is(err, auth.ErrUnauthenticated) { + return + } + if time.Now().After(deadline) { + t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance) + } + time.Sleep(100 * time.Millisecond) } } diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh index 624ede52..d223fc21 100755 --- a/scripts/bench/failover.sh +++ b/scripts/bench/failover.sh @@ -14,6 +14,13 @@ # faults.json (when and what was stopped), compose.log (the balancer's # DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. # Runs on the current stand (`make up`) and leaves every replica running. +# +# Every time used for the analysis comes from the Docker VM's clock: the log +# timestamps, the container's stop and start times from `docker inspect`, and +# the generator's own times. The host clock of Windows runs apart from it by +# a fraction of a second, a large share of a 1.5 s detection window; host +# times are kept for reference only. With `stop`, the moment SIGTERM arrived +# is the replica's own "shutdown started" line in compose.log. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -37,7 +44,8 @@ alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut - meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ "fault_at_s=$fault_at" "restore_at_s=$restore_at" sh scripts/instances.sh >"$dir/instances-before.txt" -since=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & @@ -45,17 +53,18 @@ generator=$! # The generator seeds two users first; its timeline starts after that. The # fault times below are absolute, as is started_at in timeline.json. sleep "$fault_at" -fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker "$kind" "$target" >/dev/null -fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target") sleep $((restore_at - fault_at)) -restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker compose start api >/dev/null 2>&1 +started=$(docker inspect --format '{{.State.StartedAt}}' "$target") wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" -printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \ - "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json" -docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1 +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json" +docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir" diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh index 1bab9a47..2390155a 100755 --- a/scripts/bench/rollout-under-load.sh +++ b/scripts/bench/rollout-under-load.sh @@ -8,7 +8,14 @@ # # Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default # evidence/rollout). Results in /: timeline.json, seconds.csv, -# rollout.log, update.json (when it ran), compose.log, instances-*.txt. +# rollout.log, update.json (when it ran, which containers it replaced), +# compose.log, instances-*.txt. +# +# Times for the analysis come from the Docker VM's clock, as the generator's +# do: log timestamps and the start times of the new containers. The host +# clock of Windows runs apart from it and is kept for reference only. The +# old replicas are removed by the update; their stop shows in compose.log as +# their "shutdown started" lines. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -27,21 +34,27 @@ docker compose up -d --wait >/dev/null 2>&1 docker compose build api >/dev/null 2>&1 meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" sh scripts/instances.sh >"$dir/instances-before.txt" -since=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) +old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ') BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! sleep "$update_at" -started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) status=0 sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? -finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" -printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \ - "$started" "$finished" "$status" >"$dir/update.json" -docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +new="" +for id in $(docker compose ps -q api); do + new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, " +done +printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \ + "$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json" +docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index df62c9e7..11d98aa8 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -47,8 +47,11 @@ grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool stats=$! sample_postgres "$dir/postgres.jsonl" & pg=$! + # stdin is closed: `compose run` is interactive by default and would + # otherwise read the rest of the series file this loop is reading. BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ - -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log" + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 /dev/null || true wait "$stats" "$pg" 2>/dev/null || true cat "$dir/loadgen.log" | grep '^step' || true diff --git a/scripts/rollout.sh b/scripts/rollout.sh index 1a223a83..c46e52c4 100755 --- a/scripts/rollout.sh +++ b/scripts/rollout.sh @@ -6,6 +6,10 @@ # removes the new ones and leaves the old ones serving, so the stand is never # left with a mix of versions. # +# Every replica must be running. With a stopped one (say, after stop-one), +# docker rollout only replaces the stopped one and reports success, which +# would leave old and new versions side by side; the script refuses instead. +# # The migrations of the new version run first, once, while the old replicas # serve: they only add to the schema. docker rollout scales with `compose up`, # which also starts the dependencies of the API and would rerun the old, @@ -15,6 +19,12 @@ set -eu cd "$(dirname "$0")/.." +idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api) +if [ -n "$idle" ]; then + echo "some API replicas are not running; start them first: docker compose start api" >&2 + exit 2 +fi + version=v0.14 checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b tool=bin/docker-rollout diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index d9b32e1d..1e590016 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -98,7 +98,9 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { if n := fired.Load(); n < 45 || n > 50 { t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) } - if took := time.Since(start); took > 1500*time.Millisecond { + // About 1.2 s; the bound leaves room for -race on a shared CI runner and + // still fails a closed loop, which would need 50 x 200 ms = 10 s. + if took := time.Since(start); took > 2500*time.Millisecond { t.Fatalf("open loop took %v, want about 1.2 s", took) } } -- 2.54.0 From 7fbd781ca3122f4650bc41ca6c6efa8ac38b29b3 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:19:49 +0300 Subject: [PATCH 07/21] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E=20=D0=BA=D0=BE?= =?UTF-8?q?=D0=B4=D0=B0:=20=D1=83=D0=B1=D1=80=D0=B0=D1=82=D1=8C=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2=D1=82=D0=BE=D1=80=20=D0=B2=20AppendOnce=20=D0=B7?= =?UTF-8?q?=D0=B0=20=D1=81=D1=87=D1=91=D1=82=20=D0=B7=D0=B0=D0=BF=D0=B0?= =?UTF-8?q?=D1=81=D0=B0=20=D1=83=20=D1=87=D0=B8=D1=81=D1=82=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=B9,=20=D0=B2=D1=8B=D0=BD?= =?UTF-8?q?=D0=B5=D1=81=D1=82=D0=B8=20MessagePayload=20=D0=B8=20SameConten?= =?UTF-8?q?t,=20=D0=BF=D1=80=D0=B8=D0=BD=D1=8F=D1=82=D1=8C=20X-Request-Id?= =?UTF-8?q?=20=D0=BE=D1=82=20=D0=B1=D0=B0=D0=BB=D0=B0=D0=BD=D1=81=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D1=89=D0=B8=D0=BA=D0=B0,=20=D1=83=D0=BF?= =?UTF-8?q?=D1=80=D0=BE=D1=81=D1=82=D0=B8=D1=82=D1=8C=20=D1=84=D0=BE=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D1=8B=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0=D1=87=D0=B8?= =?UTF-8?q?=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80=D0=B0,=20=D1=81=D0=BE?= =?UTF-8?q?=D0=BA=D1=80=D0=B0=D1=82=D0=B8=D1=82=D1=8C=20=D0=BA=D0=BE=D0=BC?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/migrate/main.go | 15 +--- cmd/server/main.go | 34 +++------ internal/app/app.go | 7 +- internal/app/app_test.go | 2 - internal/app/auth_test.go | 7 +- internal/app/idempotency_test.go | 5 +- internal/auth/service_test.go | 3 +- internal/chat/cleanup.go | 5 +- internal/chat/handler.go | 7 +- internal/chat/message.go | 6 ++ internal/chat/message_test.go | 13 ++++ internal/chat/repository.go | 10 ++- internal/chat/service.go | 42 ++++++----- internal/chat/service_test.go | 39 +++++++--- internal/chat/store/export_test.go | 9 --- internal/chat/store/idempotency_test.go | 24 ------- internal/chat/store/postgres.go | 79 +++++++-------------- internal/platform/config/config.go | 49 +++++++------ internal/platform/config/config_test.go | 6 +- internal/platform/httpx/httpx_test.go | 20 ++++-- internal/platform/httpx/middleware.go | 30 ++++---- internal/platform/ident/ident.go | 5 +- internal/platform/postgres/postgres.go | 14 ++-- internal/platform/postgres/postgres_test.go | 2 - internal/server/server.go | 21 ++---- 25 files changed, 207 insertions(+), 247 deletions(-) delete mode 100644 internal/chat/store/export_test.go diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 201cec5d..851b2e85 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,6 +1,5 @@ -// Command migrate applies the embedded schema migrations once and exits. It -// runs as a one-shot service before the API replicas start, so the replicas -// never race each other over the schema and need no rights to change it. +// Command migrate applies the embedded schema migrations once and exits, so +// the API replicas never race each other over the schema. package main import ( @@ -24,7 +23,7 @@ func main() { fmt.Fprintln(os.Stderr, "configuration error:", err) os.Exit(2) } - log := newLogger(cfg) + log := cfg.NewLogger() ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() @@ -48,11 +47,3 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error { log.Info("migrations applied", "applied_migrations", applied) return nil } - -func newLogger(cfg config.Migrate) *slog.Logger { - opts := &slog.HandlerOptions{Level: cfg.LogLevel} - if cfg.Env == "dev" { - return slog.New(slog.NewTextHandler(os.Stdout, opts)) - } - return slog.New(slog.NewJSONHandler(os.Stdout, opts)) -} diff --git a/cmd/server/main.go b/cmd/server/main.go index 6bf7d66b..4947a381 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -25,7 +25,6 @@ import ( "messenger/internal/server" ) -// keyCleanupEvery is how often each replica deletes expired Idempotency-Keys. const keyCleanupEvery = 10 * time.Minute func main() { @@ -34,10 +33,9 @@ func main() { fmt.Fprintln(os.Stderr, "configuration error:", err) os.Exit(2) } - // Every log line carries the alias that this process sends as - // X-Instance-Id, so a response can be matched to its replica's log. + alias := ident.NewAlias() - log := newLogger(cfg).With("instance", alias) + log := cfg.NewLogger().With("instance", alias) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() @@ -49,8 +47,7 @@ func main() { } // run connects the dependencies before it listens, so the port accepts -// connections only once they answer. The schema is applied beforehand by the -// separate migrate command, once per deployment rather than by every replica. +// connections only once they answer. func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error { pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns) if err != nil { @@ -88,12 +85,8 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) publishDebugVars(alias, pool) } - // The background work ends with the server: on a signal, and also when - // the server fails, so the deferred Wait cannot block the exit. - var background sync.WaitGroup - defer background.Wait() // before the deferred pool.Close ctx, cancel := context.WithCancel(ctx) - defer cancel() // before Wait + var background sync.WaitGroup background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) if debugLn != nil { background.Go(func() { @@ -103,13 +96,13 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) }) } - return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) + err = server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) + cancel() + background.Wait() + return err } -// publishDebugVars exposes, next to the runtime's memstats, what the scaling -// study needs from a replica: who it is, how many CPUs Go uses inside the -// container's limit, and how often requests had to wait for a pooled -// database connection. +// publishDebugVars adds to expvar what the load tests read from a replica. func publishDebugVars(alias string, pool *pgxpool.Pool) { expvar.NewString("instance").Set(alias) expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) })) @@ -125,14 +118,7 @@ func publishDebugVars(alias string, pool *pgxpool.Pool) { "empty_acquire_count": s.EmptyAcquireCount(), "empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(), "canceled_acquire_count": s.CanceledAcquireCount(), + "new_conns_count": s.NewConnsCount(), } })) } - -func newLogger(cfg config.Config) *slog.Logger { - opts := &slog.HandlerOptions{Level: cfg.LogLevel} - if cfg.Env == "dev" { - return slog.New(slog.NewTextHandler(os.Stdout, opts)) - } - return slog.New(slog.NewJSONHandler(os.Stdout, opts)) -} diff --git a/internal/app/app.go b/internal/app/app.go index 24dd4c16..720a9ae5 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -39,10 +39,8 @@ type Deps struct { MaxBodyBytes int64 // RequestTimeout bounds every API request; zero means the default. RequestTimeout time.Duration - // InstanceID is sent as X-Instance-Id on every response. - InstanceID string - // IdempotencyTTL is how long a message Idempotency-Key is remembered; - // zero means the default. + InstanceID string + // IdempotencyTTL zero means the contract's 24 hours. IdempotencyTTL time.Duration } @@ -50,7 +48,6 @@ type Deps struct { // request that gives up on a dependency can still write its 503. const defaultRequestTimeout = 5 * time.Second -// defaultIdempotencyTTL is the contract's retention of message keys. const defaultIdempotencyTTL = 24 * time.Hour func New(d Deps) *App { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 622d46c6..02f8a33b 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -52,7 +52,6 @@ func TestMain(m *testing.M) { const testPassword = "correct-horse-battery" -// testInstance is the alias of the in-process application under test. const testInstance = "00000000000000a1" // errorBody mirrors the contract error envelope. @@ -245,7 +244,6 @@ func (ts *tester) message(token, chatID, text string) chat.MessageResponse { return decode[chat.MessageResponse](ts.t, res) } -// freshKey is the header of a first attempt: a key never used before. func freshKey() map[string]string { return map[string]string{"Idempotency-Key": uuid.NewString()} } diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go index 2ed17d62..d0d4819b 100644 --- a/internal/app/auth_test.go +++ b/internal/app/auth_test.go @@ -146,11 +146,8 @@ func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") } -// The expiry is an absolute time written by the application and enforced by -// Valkey's clock. Here the test runs on the Windows host and Valkey in the -// Docker VM, whose clock lags by a fraction of a second, so the test waits -// for the 401 with a tolerance instead of sleeping exactly one TTL. On the -// stand both run in the same VM. +// Valkey enforces the expiry by the Docker VM clock, which may lag this host +// by a fraction of a second, so the 401 is awaited with a tolerance. func TestSessionExpiresAfterTTL(t *testing.T) { const ttl, clockTolerance = 2 * time.Second, 3 * time.Second ts := newTesterWithTTL(t, ttl) diff --git a/internal/app/idempotency_test.go b/internal/app/idempotency_test.go index 9da80609..e213baab 100644 --- a/internal/app/idempotency_test.go +++ b/internal/app/idempotency_test.go @@ -101,9 +101,8 @@ func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) { ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found") } -// Two applications over the same stores stand for two replicas behind the -// balancer: retries of one key alternate between them and still create one -// message. +// Two applications over the same stores stand for two replicas: retries of one +// key alternate between them and still create one message. func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) { ts := newTester(t) alice := ts.user("alice", "A") diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go index b5c8173f..e3cd3fb2 100644 --- a/internal/auth/service_test.go +++ b/internal/auth/service_test.go @@ -129,8 +129,7 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) { } } -// ExpiresAt is written by this process's clock and enforced by Valkey's. In -// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host +// Valkey enforces the expiry by the Docker VM clock, which may lag this host // by a fraction of a second, so the expiry is awaited with a tolerance. func TestSessionExpiresAfterTTL(t *testing.T) { const clockTolerance = 3 * time.Second diff --git a/internal/chat/cleanup.go b/internal/chat/cleanup.go index 1fd1d050..13db162e 100644 --- a/internal/chat/cleanup.go +++ b/internal/chat/cleanup.go @@ -6,10 +6,7 @@ import ( "time" ) -// RunKeyCleanup deletes expired Idempotency-Keys every period until ctx is -// cancelled. Every replica runs it: the deletes skip rows another replica or -// a request holds, so concurrent runs share the work instead of waiting, and -// a failed run is simply retried at the next tick. +// RunKeyCleanup purges expired Idempotency-Keys every period until ctx is done. func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) { ticker := time.NewTicker(every) defer ticker.Stop() diff --git a/internal/chat/handler.go b/internal/chat/handler.go index b70737bf..c913393e 100644 --- a/internal/chat/handler.go +++ b/internal/chat/handler.go @@ -144,7 +144,7 @@ func (h *Handler) send(c *gin.Context, actor uuid.UUID) { httpx.Error(c, err) return } - message, err := h.chats.Send(c.Request.Context(), actor, chatID, key, req.Text) + message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: req.Text}) if err != nil { httpx.Error(c, err) return @@ -173,10 +173,7 @@ func (h *Handler) history(c *gin.Context, actor uuid.UUID) { var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) -// idempotencyKey reads the Idempotency-Key that names one send attempt. It -// is part of this endpoint's input contract, so it is checked here, with -// the path and the body, and before access: the answer to a malformed key -// is the same for any chat. Two headers are ambiguous and rejected. +// idempotencyKey rejects two values of the header as ambiguous. func idempotencyKey(c *gin.Context) (string, error) { values := c.Request.Header.Values("Idempotency-Key") if len(values) != 1 || !keyPattern.MatchString(values[0]) { diff --git a/internal/chat/message.go b/internal/chat/message.go index c28f6d7d..06a758bc 100644 --- a/internal/chat/message.go +++ b/internal/chat/message.go @@ -42,3 +42,9 @@ func NewMessage(id, chatID, senderID uuid.UUID, text string) (Message, error) { CreatedAt: ident.TimeOf(id), }, nil } + +// SameContent reports whether two messages say the same thing, whatever their +// ids and times. +func (m Message) SameContent(other Message) bool { + return m.Text == other.Text +} diff --git a/internal/chat/message_test.go b/internal/chat/message_test.go index 3cb76738..42762154 100644 --- a/internal/chat/message_test.go +++ b/internal/chat/message_test.go @@ -45,3 +45,16 @@ func TestNewMessageKeepsSurroundingWhitespace(t *testing.T) { t.Fatalf("text must be stored verbatim, got %q", m.Text) } } + +func TestSameContentComparesWhatTheSenderWrote(t *testing.T) { + chatID, alice := ident.New(), ident.New() + first, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + retry, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + other, _ := chat.NewMessage(ident.New(), chatID, alice, "привет!") + if !first.SameContent(retry) { + t.Fatal("messages with the same text must have the same content") + } + if first.SameContent(other) { + t.Fatal("messages with different text must differ") + } +} diff --git a/internal/chat/repository.go b/internal/chat/repository.go index 83fd6d37..47d929d7 100644 --- a/internal/chat/repository.go +++ b/internal/chat/repository.go @@ -24,13 +24,11 @@ type ChatRepository interface { } type MessageRepository interface { - // AppendOnce stores message under the sender's key in its chat, unless - // the key was used there within ttl; it returns the message the key is - // bound to: the given one or the earlier one. + // AppendOnce stores message under the sender's key unless the key was + // used in this chat within ttl, and returns the message the key is bound + // to: the given one or the earlier one. AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error) - // DeleteExpiredKeys removes up to limit keys older than ttl and reports - // how many it removed. - DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) + DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) // ListByChat returns up to limit messages ordered by (created_at, id), // strictly after the message identified by after when set. An unknown // chat yields an empty history. diff --git a/internal/chat/service.go b/internal/chat/service.go index b89fdb3d..ac71e96f 100644 --- a/internal/chat/service.go +++ b/internal/chat/service.go @@ -12,6 +12,13 @@ import ( "messenger/internal/platform/pagination" ) +const ( + keyPurgeBatch = 1000 + // keyPurgeGrace keeps a key past its TTL, so a key that a request has just + // found valid cannot vanish before the request reads its message back. + keyPurgeGrace = time.Minute +) + // UserLookup is what this module needs from the users module: one batched // existence check. It is declared here, on the consumer side, so this // package never imports the users package and tests substitute a map. @@ -26,8 +33,8 @@ type Service struct { keyTTL time.Duration } -// NewService builds the chats service; keyTTL is how long a message -// Idempotency-Key stays bound to its message. +// NewService takes keyTTL, how long an Idempotency-Key stays bound to its +// message. func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service { return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL} } @@ -66,38 +73,37 @@ func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, l return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil } -// Send stores a message under the sender's Idempotency-Key. Access and the -// text are checked first, on every attempt: a retry is a new request that -// must still be authorised and valid. A key already bound to a message with -// the same text returns that message; with a different text it is a conflict. -func (s *Service) Send(ctx context.Context, actor, chatID uuid.UUID, key, text string) (Message, error) { - if err := s.requireMember(ctx, actor, chatID); err != nil { +// MessagePayload is one send attempt: the message and its Idempotency-Key. +type MessagePayload struct { + ChatID uuid.UUID + IdempotencyKey string + Text string +} + +// Send checks access and the text on every attempt, a replay included, as the +// contract requires of a repeat. +func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) { + if err := s.requireMember(ctx, actor, p.ChatID); err != nil { return Message{}, err } - message, err := NewMessage(ident.New(), chatID, actor, text) + message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text) if err != nil { return Message{}, err } - stored, err := s.messages.AppendOnce(ctx, message, key, s.keyTTL) + stored, err := s.messages.AppendOnce(ctx, message, p.IdempotencyKey, s.keyTTL) if err != nil { return Message{}, err } - if stored.ID != message.ID && stored.Text != message.Text { + if !stored.SameContent(message) { return Message{}, ErrKeyReused } return stored, nil } -// keyPurgeBatch bounds one delete statement, so the cleanup never holds many -// row locks or runs long. -const keyPurgeBatch = 1000 - -// PurgeExpiredKeys deletes every key older than the retention period, batch -// by batch, and reports how many it deleted. func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) { var total int64 for { - n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL, keyPurgeBatch) + n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL+keyPurgeGrace, keyPurgeBatch) total += n if err != nil { return total, err diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go index 3cf95f83..1098708c 100644 --- a/internal/chat/service_test.go +++ b/internal/chat/service_test.go @@ -190,15 +190,15 @@ func TestSendChecksAccessBeforeValidation(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = f.svc.Send(f.ctx, f.eve, c.ID, "k1", " ") + _, err = f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: " "}) if !errors.Is(err, chat.ErrNotFound) { t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err) } - _, err = f.svc.Send(f.ctx, f.bob, c.ID, "k2", " ") + _, err = f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: " "}) if apperr.KindOf(err) != apperr.KindInvalid { t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err)) } - msg, err := f.svc.Send(f.ctx, f.bob, c.ID, "k3", "Привет 👋") + msg, err := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k3", Text: "Привет 👋"}) if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" { t.Fatalf("Send = %+v, %v", msg, err) } @@ -210,8 +210,8 @@ func TestHistoryPagesAndHidesForeignChat(t *testing.T) { if err != nil { t.Fatal(err) } - first, _ := f.svc.Send(f.ctx, f.alice, c.ID, "k1", "1") - second, _ := f.svc.Send(f.ctx, f.bob, c.ID, "k2", "2") + first, _ := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: "1"}) + second, _ := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: "2"}) page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1) if err != nil { @@ -247,7 +247,7 @@ func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID { func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message { t.Helper() - m, err := f.svc.Send(f.ctx, actor, c, key, text) + m, err := f.svc.Send(f.ctx, actor, chat.MessagePayload{ChatID: c, IdempotencyKey: key, Text: text}) if err != nil { t.Fatal(err) } @@ -268,7 +268,7 @@ func TestSendConflictsOnDifferentBody(t *testing.T) { f := newFixture(t) c := f.newChat(t, f.alice, f.bob) f.send(t, f.alice, c, "k", "первое") - _, err := f.svc.Send(f.ctx, f.alice, c, "k", "другое") + _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "другое"}) if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict { t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err) } @@ -292,7 +292,7 @@ func TestSendChecksAccessBeforeKey(t *testing.T) { f := newFixture(t) c := f.newChat(t, f.alice, f.bob) first := f.send(t, f.alice, c, "k", "секрет") - if _, err := f.svc.Send(f.ctx, f.eve, c, "k", "секрет"); !errors.Is(err, chat.ErrNotFound) { + if _, err := f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "секрет"}); !errors.Is(err, chat.ErrNotFound) { t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err) } if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID { @@ -304,7 +304,7 @@ func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) { f := newFixture(t) c := f.newChat(t, f.alice, f.bob) f.send(t, f.alice, c, "k", "текст") - if _, err := f.svc.Send(f.ctx, f.alice, c, "k", " "); apperr.KindOf(err) != apperr.KindInvalid { + if _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: " "}); apperr.KindOf(err) != apperr.KindInvalid { t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err)) } } @@ -334,6 +334,27 @@ func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) { } } +// A key the claim has just seen valid must survive until the replay lookup, +// so the cleanup leaves keys alone for a while after they expire. +func TestPurgeExpiredKeysWaitsAGraceAfterTTL(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "just-expired", "1") + f.send(t, f.alice, c, "long-expired", "2") + if _, err := db.Pool.Exec(f.ctx, ` + UPDATE message_idempotency_keys + SET created_at = now() - interval '1 hour' - CASE key WHEN 'just-expired' THEN interval '10 seconds' ELSE interval '10 minutes' END`); err != nil { + t.Fatal(err) + } + if deleted, err := f.svc.PurgeExpiredKeys(f.ctx); err != nil || deleted != 1 { + t.Fatalf("PurgeExpiredKeys = %d, %v; want only the long expired key", deleted, err) + } + var left string + if err := db.Pool.QueryRow(f.ctx, `SELECT key FROM message_idempotency_keys`).Scan(&left); err != nil || left != "just-expired" { + t.Fatalf("left %q, %v; want just-expired", left, err) + } +} + func TestRunKeyCleanupPurgesAndStops(t *testing.T) { f := newFixture(t) c := f.newChat(t, f.alice) diff --git a/internal/chat/store/export_test.go b/internal/chat/store/export_test.go deleted file mode 100644 index 528730c4..00000000 --- a/internal/chat/store/export_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package store - -// SetAfterClaim installs a hook that runs between the claiming statement and -// the replay lookup of AppendOnce; it returns a function restoring the old one. -func SetAfterClaim(hook func()) (restore func()) { - old := afterClaim - afterClaim = hook - return func() { afterClaim = old } -} diff --git a/internal/chat/store/idempotency_test.go b/internal/chat/store/idempotency_test.go index 72f8714e..2b489947 100644 --- a/internal/chat/store/idempotency_test.go +++ b/internal/chat/store/idempotency_test.go @@ -9,7 +9,6 @@ import ( "github.com/google/uuid" "messenger/internal/chat" - "messenger/internal/chat/store" ) const keyTTL = time.Hour @@ -168,29 +167,6 @@ func TestCleanerSkipsRowLockedByRequest(t *testing.T) { } } -// The key is alive when the claim runs but deleted before the replay lookup, -// as by a cleaner whose clock is a moment later at the TTL boundary. The -// operation starts over and stores the new message. -func TestAppendOnceRetriesWhenKeyVanishes(t *testing.T) { - f := newFixture(t) - alice := f.mustUser(t) - c := f.mustChat(t, alice) - f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") - var once sync.Once - restore := store.SetAfterClaim(func() { - once.Do(func() { - if _, err := db.Pool.Exec(context.Background(), `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil { - t.Error(err) - } - }) - }) - defer restore() - next := f.newMessage(t, c.ID, alice) - if got := f.appendOnce(t, next, "k1"); got.ID != next.ID { - t.Fatalf("got %v, want the new message %v", got.ID, next.ID) - } -} - func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) { f := newFixture(t) alice := f.mustUser(t) diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go index d93f6592..6554bf9f 100644 --- a/internal/chat/store/postgres.go +++ b/internal/chat/store/postgres.go @@ -118,16 +118,9 @@ func NewMessages(pool *pgxpool.Pool) *Messages { return &Messages{pool: pool} } -// afterClaim runs between the claiming statement and the replay lookup of -// AppendOnce; tests use it to delete the key in that window. -var afterClaim func() - -// claimSQL stores the key and the message in one statement. A key older than -// the retention period is taken over for the new message. The key row is -// inserted first: when it already exists and is still valid, claimed is -// empty and so is the message insert. One statement holds the key's lock -// only while the server executes it, never across a round trip to a replica -// that might die in between. +// claimSQL binds the key and inserts the message in one statement, so the +// key's lock never spans a round trip. An expired key is taken over; a valid +// one leaves claimed, and so the message insert, empty. const claimSQL = ` WITH claimed AS ( INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) @@ -139,51 +132,33 @@ const claimSQL = ` INSERT INTO messages (chat_id, id, sender_id, text, created_at) SELECT $2, message_id, $1, $6, $7 FROM claimed` -// AppendOnce stores m under key unless the sender already used the key in -// this chat within ttl; it returns the message the key is bound to, m itself -// or the earlier one. Deciding whether the earlier message answers the same -// request is the caller's job. -// -// The lookup of an existing key is a second statement: within the claiming -// one, a plain SELECT would see the snapshot taken before a concurrent -// request committed the key. Between the two statements the cleaner may -// delete a key at the ttl boundary; the operation then starts over once, -// which stores m under the now free key. +// The bound message is read by a second statement: the claim's snapshot +// cannot see a key that a concurrent request has just committed. func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { - for range 2 { - tag, err := r.pool.Exec(ctx, claimSQL, - m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) - if err != nil { - return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) - } - if tag.RowsAffected() == 1 { - return m, nil - } - if afterClaim != nil { - afterClaim() - } - var stored chat.Message - err = r.pool.QueryRow(ctx, ` - SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at - FROM message_idempotency_keys k - JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id - WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, - m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) - if errors.Is(err, pgx.ErrNoRows) { - continue - } - if err != nil { - return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) - } - return stored, nil + tag, err := r.pool.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return chat.Message{}, fmt.Errorf("insert message: %w", postgres.Classify(err)) } - return chat.Message{}, errors.New("idempotency key vanished twice between claim and lookup") + if tag.RowsAffected() == 1 { + return m, nil + } + var stored chat.Message + err = r.pool.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key).Scan(&stored.ID, &stored.ChatID, &stored.SenderID, &stored.Text, &stored.CreatedAt) + if err != nil { + return chat.Message{}, fmt.Errorf("select message by key: %w", postgres.Classify(err)) + } + return stored, nil } -// DeleteExpiredKeys removes up to limit keys older than ttl. SKIP LOCKED lets -// replicas cleaning at the same time, and requests holding a key, pass each -// other instead of waiting. -func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, limit int) (int64, error) { +// DeleteExpiredKeys skips rows locked by another cleaner or a request rather +// than waiting for them. +func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) { tag, err := r.pool.Exec(ctx, ` DELETE FROM message_idempotency_keys WHERE (user_id, chat_id, key) IN ( @@ -192,7 +167,7 @@ func (r *Messages) DeleteExpiredKeys(ctx context.Context, ttl time.Duration, lim ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED)`, - interval(ttl), limit) + interval(olderThan), limit) if err != nil { return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err)) } diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 0dcc0d9d..49e191f3 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -15,31 +15,41 @@ import ( "time" ) +// Logging selects the log format, text for dev and JSON otherwise, and the +// level. +type Logging struct { + Env string + Level slog.Level +} + +func (l Logging) NewLogger() *slog.Logger { + opts := &slog.HandlerOptions{Level: l.Level} + if l.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} + type Config struct { + Logging Host string Port int - Env string - LogLevel slog.Level ShutdownTimeout time.Duration MaxBodyBytes int64 DatabaseURL string SessionStoreURL string SessionTTL time.Duration - // DrainDelay is how long a stopping process keeps serving with readiness - // already off, so the load balancer can take it out of rotation before - // the listener closes. It must exceed the balancer's detection time. + // DrainDelay keeps a stopping process serving with readiness off; it must + // exceed the balancer's detection time. DrainDelay time.Duration IdempotencyTTL time.Duration DBMaxConns int32 - // DebugAddr enables the internal expvar listener; empty disables it. + // DebugAddr enables the internal expvar listener when set. DebugAddr string } -// Migrate is the configuration of the one-shot migration command, which -// needs the database only. type Migrate struct { - Env string - LogLevel slog.Level + Logging DatabaseURL string } @@ -51,8 +61,6 @@ func Load(lookup Lookup) (Config, error) { cfg := Config{ Host: "0.0.0.0", Port: 8080, - Env: "lab", - LogLevel: slog.LevelInfo, ShutdownTimeout: 8 * time.Second, MaxBodyBytes: 64 * 1024, SessionTTL: time.Hour, @@ -74,7 +82,7 @@ func Load(lookup Lookup) (Config, error) { cfg.Port = port } var err error - if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + if cfg.Logging, err = loadLogging(lookup); err != nil { return Config{}, err } if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { @@ -135,9 +143,9 @@ func Load(lookup Lookup) (Config, error) { } func LoadMigrate(lookup Lookup) (Migrate, error) { - cfg := Migrate{Env: "lab", LogLevel: slog.LevelInfo} + var cfg Migrate var err error - if cfg.Env, cfg.LogLevel, err = logging(lookup, cfg.Env, cfg.LogLevel); err != nil { + if cfg.Logging, err = loadLogging(lookup); err != nil { return Migrate{}, err } if cfg.DatabaseURL, err = secret(lookup, "DATABASE_URL_FILE"); err != nil { @@ -146,16 +154,17 @@ func LoadMigrate(lookup Lookup) (Migrate, error) { return cfg, nil } -func logging(lookup Lookup, env string, level slog.Level) (string, slog.Level, error) { +func loadLogging(lookup Lookup) (Logging, error) { + l := Logging{Env: "lab", Level: slog.LevelInfo} if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { - env = v + l.Env = v } if v, ok := lookup("APP_LOG_LEVEL"); ok { - if err := level.UnmarshalText([]byte(v)); err != nil { - return "", 0, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + if err := l.Level.UnmarshalText([]byte(v)); err != nil { + return Logging{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) } } - return env, level, nil + return l, nil } // secret reads the file named by the variable; surrounding whitespace, diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go index 4900ff3d..e58ffe19 100644 --- a/internal/platform/config/config_test.go +++ b/internal/platform/config/config_test.go @@ -53,7 +53,7 @@ func TestDefaults(t *testing.T) { if cfg.Addr() != "0.0.0.0:8080" { t.Fatalf("Addr = %q", cfg.Addr()) } - if cfg.Env != "lab" || cfg.LogLevel != slog.LevelInfo { + if cfg.Env != "lab" || cfg.Level != slog.LevelInfo { t.Fatalf("cfg = %+v", cfg) } if cfg.ShutdownTimeout != 8*time.Second || cfg.MaxBodyBytes != 65536 { @@ -110,7 +110,7 @@ func TestOverrides(t *testing.T) { if err != nil { t.Fatalf("Load error: %v", err) } - if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.LogLevel != slog.LevelDebug { + if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.Level != slog.LevelDebug { t.Fatalf("cfg = %+v", cfg) } if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { @@ -168,7 +168,7 @@ func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) { if err != nil { t.Fatalf("LoadMigrate error: %v", err) } - if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.LogLevel != slog.LevelWarn { + if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.Level != slog.LevelWarn { t.Fatalf("cfg = %+v", cfg) } if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") { diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go index 66244ad7..30c64aca 100644 --- a/internal/platform/httpx/httpx_test.go +++ b/internal/platform/httpx/httpx_test.go @@ -193,10 +193,22 @@ func TestSuccessResponseHeadersAndEscaping(t *testing.T) { } } -func TestRequestIDIsGeneratedNotEchoed(t *testing.T) { - rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": "client-chosen"}) - if got := rec.Header().Get("X-Request-Id"); got == "" || got == "client-chosen" { - t.Fatalf("X-Request-Id = %q, want a server generated value", got) +// The balancer sets X-Request-Id on every request, overwriting what the +// client sent; the API keeps it so that one id spans both logs. +func TestRequestIDFromTheBalancerIsKept(t *testing.T) { + const fromLB = "0123456789abcdef0123456789abcdef" + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": fromLB}) + if got := rec.Header().Get("X-Request-Id"); got != fromLB { + t.Fatalf("X-Request-Id = %q, want %q", got, fromLB) + } +} + +func TestMalformedRequestIDIsReplaced(t *testing.T) { + for _, bad := range []string{"has space", "semi;colon", "tab\there", strings.Repeat("a", 65)} { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": bad}) + if got := rec.Header().Get("X-Request-Id"); got == "" || got == bad { + t.Errorf("X-Request-Id for %q = %q, want a generated value", bad, got) + } } } diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go index a0a9cc6d..b4844e9d 100644 --- a/internal/platform/httpx/middleware.go +++ b/internal/platform/httpx/middleware.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "net/http" + "regexp" "runtime/debug" "time" @@ -22,22 +23,17 @@ func instance(id string) gin.HandlerFunc { } } -// requestID assigns a fresh identifier to every request. -// -// An incoming X-Request-Id is deliberately not reused. This service is -// client-facing: there is no trusted upstream that could have minted the -// value, so honouring it would let any caller pick an arbitrary string that -// ends up in logs (log injection, unbounded length) or collides with another -// request and corrupts log correlation. Reusing the header is the right -// choice only when the caller is a trusted proxy or a sibling service that -// propagates one id across a chain of calls. +var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// requestID keeps the X-Request-Id set by the load balancer, which overwrites +// any value a client sends; a malformed value is replaced, so a direct caller +// cannot inject into the logs. func requestID(log *slog.Logger) gin.HandlerFunc { return func(c *gin.Context) { - var raw [16]byte - if _, err := rand.Read(raw[:]); err != nil { - panic("httpx: random source unavailable: " + err.Error()) + id := c.GetHeader("X-Request-Id") + if !requestIDPattern.MatchString(id) { + id = newRequestID() } - id := hex.EncodeToString(raw[:]) c.Set(keyRequestID, id) c.Set(keyLogger, log) c.Writer.Header().Set("X-Request-Id", id) @@ -45,6 +41,14 @@ func requestID(log *slog.Logger) gin.HandlerFunc { } } +func newRequestID() string { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + panic("httpx: random source unavailable: " + err.Error()) + } + return hex.EncodeToString(raw[:]) +} + // accessLog writes one line per request at a level that follows the // outcome (2xx debug, 4xx info, 5xx error), so the ordinary log level decides // how much traffic is recorded. Headers, query strings and bodies are never diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go index b70efbe9..f50f946a 100644 --- a/internal/platform/ident/ident.go +++ b/internal/platform/ident/ident.go @@ -27,9 +27,8 @@ func New() uuid.UUID { return id } -// NewAlias names one server process for X-Instance-Id. It is random rather -// than the hostname, so the header reveals nothing about the infrastructure, -// and a restarted process gets a new one. +// NewAlias names one server process for X-Instance-Id; random rather than the +// hostname, so the header reveals nothing about the infrastructure. func NewAlias() string { var raw [8]byte _, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24. diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go index 6409553e..d9a7d653 100644 --- a/internal/platform/postgres/postgres.go +++ b/internal/platform/postgres/postgres.go @@ -2,8 +2,7 @@ // // Server-side timeouts are constants rather than settings: they bound how // long one request can hold a connection, and the request deadline of the API -// is chosen against them. The pool size is a setting, because the scaling -// study varies it. +// is chosen against them. package postgres import ( @@ -75,21 +74,20 @@ func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, er return pool, nil } -// Migrate applies pending migrations and returns the versions it applied. A -// session advisory lock serialises concurrent starts, so several API -// replicas may call it at once; applied versions are recorded, which makes a -// repeated run a no-op. +// Migrate applies pending migrations and returns their versions; a session +// advisory lock makes overlapping runs wait for each other. func Migrate(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) { - db := stdlib.OpenDBFromPool(pool) - defer db.Close() locker, err := lock.NewPostgresSessionLocker() if err != nil { return nil, fmt.Errorf("migration lock: %w", err) } + db := stdlib.OpenDBFromPool(pool) provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.FS, goose.WithSessionLocker(locker)) if err != nil { + db.Close() return nil, fmt.Errorf("load migrations: %w", err) } + defer provider.Close() results, err := provider.Up(ctx) if err != nil { return nil, fmt.Errorf("apply migrations: %w", err) diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go index 76daaf7c..6f56fb39 100644 --- a/internal/platform/postgres/postgres_test.go +++ b/internal/platform/postgres/postgres_test.go @@ -190,8 +190,6 @@ func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { } } -// The pool size is an experiment variable of the scaling study, so it is -// passed in rather than fixed. func TestNewPoolUsesTheRequestedSize(t *testing.T) { pool, err := postgres.NewPool(context.Background(), db.DSN, 3) if err != nil { diff --git a/internal/server/server.go b/internal/server/server.go index c33fc2b1..64f7abf8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,7 +1,5 @@ -// Package server runs the HTTP listener with the lifecycle the lab -// requires: readiness drops first, the process keeps serving for a drain -// delay while the load balancer notices, then the listener stops accepting, -// in-flight requests drain within a bounded timeout and the process exits. +// Package server runs the HTTP listener with a graceful stop behind a load +// balancer. package server import ( @@ -19,13 +17,9 @@ type Readiness interface { } // Run serves until ctx is cancelled, keeps serving for drainDelay with -// readiness off, then shuts down within shutdownTimeout. -// -// The balancer learns about a stopping replica only from its failing -// readiness checks. Closing the listener at once would let it send requests -// to a closed port, or over a kept-alive connection the server is closing, -// until the checks catch up. drainDelay must therefore exceed the balancer's -// detection time. +// readiness off, then shuts down within shutdownTimeout. The balancer learns +// of the stop only from failing readiness checks, so drainDelay must exceed +// its detection time. func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error { srv := &http.Server{ Handler: h, @@ -68,9 +62,8 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, return nil } -// RunDebug serves h, the internal metrics of the process, on ln until ctx is -// cancelled. Nothing outside the stand's network reaches it and no request -// depends on it, so it closes at once instead of draining. +// RunDebug serves the internal metrics until ctx is cancelled; no request +// depends on them, so it stops without draining. func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error { srv := &http.Server{ Handler: h, -- 2.54.0 From ce47c138e4744d81be2cf9934bcd50f59cf027ec Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:23:14 +0300 Subject: [PATCH 08/21] =?UTF-8?q?=D0=A0=D0=B0=D0=B7=D0=B4=D0=B5=D0=BB?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=81=D1=82=D0=B5=D0=BD=D0=B4=20=D0=B8=20?= =?UTF-8?q?=D0=BD=D0=B0=D1=81=D1=82=D1=80=D0=BE=D0=B9=D0=BA=D0=B8=20=D0=BD?= =?UTF-8?q?=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BE=D1=87=D0=BD=D1=8B=D1=85=20?= =?UTF-8?q?=D1=8D=D0=BA=D1=81=D0=BF=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D1=82=D0=BE=D0=B2,=20=D0=BE=D0=B3=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=87=D0=B8=D1=82=D1=8C=20=D1=87=D0=B8=D1=81=D0=BB=D0=BE=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D0=BE=D0=B2=20=D0=BD=D0=B0?= =?UTF-8?q?=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D1=83=20=D0=B2=20HAProx?= =?UTF-8?q?y=20=D0=B8=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B4=D0=B0=D0=B2=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20X-Request-Id=20=D0=BE=D1=82=20=D0=B1=D0=B0=D0=BB?= =?UTF-8?q?=D0=B0=D0=BD=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B2=20API,=20=D1=81=D0=BE=D0=BA=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 + Dockerfile | 18 ++--- compose.bench.yaml | 47 ++++++++++++ compose.yaml | 74 ++++--------------- infra/haproxy/haproxy.cfg | 49 +++++------- migrations/00002_message_idempotency_keys.sql | 12 +-- scripts/bench/lib.sh | 3 + 7 files changed, 94 insertions(+), 111 deletions(-) create mode 100644 compose.bench.yaml diff --git a/.dockerignore b/.dockerignore index c872c535..95cf4c12 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,5 @@ node_modules evidence .DS_Store compose*.yaml +bench-out +bin diff --git a/Dockerfile b/Dockerfile index f32eecf5..2a0b1902 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ -# Build stage: compile static binaries with the exact toolchain pinned in go.mod. -# Modules live in their own layer, rebuilt only when go.mod or go.sum change. -# Only the Go sources are copied after it, and the compiler's cache persists -# between builds: a change elsewhere in the repository does not trigger a full -# recompilation, which matters when a rolling update rebuilds the image. +# Modules in their own layer and a persistent build cache keep the rebuild of +# a rolling update short. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ @@ -13,8 +10,7 @@ COPY migrations ./migrations RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Load generator of the experiments, an image of its own: the API image does -# not carry it. Built only for `docker compose --profile bench`. +# The load generator gets an image of its own; the API image does not carry it. FROM build AS loadgen-build COPY tools ./tools RUN --mount=type=cache,target=/root/.cache/go-build \ @@ -26,9 +22,7 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen USER app ENTRYPOINT ["/usr/local/bin/loadgen"] -# Runtime stage, the last one so that `build: .` builds it: minimal image, -# unprivileged user, the server is PID 1. The same image runs the one-shot -# migrate service with its own entrypoint. +# Last, so that `build: .` builds it; it also runs the migrate service. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app COPY --from=build /out/server /out/migrate /usr/local/bin/ @@ -37,8 +31,8 @@ ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab EXPOSE 8080 -# The short start interval lets `compose up --wait` and the rolling update see -# a new replica as healthy about a second after it is ready. +# The start interval lets a rolling update see a new replica healthy within +# about a second. HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 ENTRYPOINT ["/usr/local/bin/server"] diff --git a/compose.bench.yaml b/compose.bench.yaml new file mode 100644 index 00000000..0a61825e --- /dev/null +++ b/compose.bench.yaml @@ -0,0 +1,47 @@ +# Load-test override of compose.yaml, used only by scripts/bench: every value +# an experiment varies, and the load generator. Defaults equal compose.yaml. +services: + lb: + environment: + LB_QUIET: ${LB_QUIET:-0} + API_MAXCONN: ${API_MAXCONN:-20} + deploy: + resources: + limits: + cpus: "${LB_CPUS:-0.5}" + + api: + environment: + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} + APP_DEBUG_ADDR: ":9090" + deploy: + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-256M} + + postgres: + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEMORY:-512M} + + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab3 + environment: + GOMEMLIMIT: 900MiB + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-2.0}" + memory: 1G diff --git a/compose.yaml b/compose.yaml index 5e1421eb..09b76e75 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,12 +1,6 @@ # Local stand: HAProxy in front of interchangeable API replicas over shared -# PostgreSQL (users, chats, messages, idempotency keys) and Valkey (sessions). -# Only the balancer is published, on loopback. Credentials are files in -# ./secrets created by `make secrets`; named volumes keep the data across -# restarts, and only `make destroy` removes them. -# -# Two networks: edge joins the balancer and the API, data joins the API, the -# migrations and the stores and has no way out. The balancer cannot reach the -# stores, and the stores cannot reach anything outside. +# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data +# network has no way out. The load-test knobs are in compose.bench.yaml. name: lab3 services: @@ -14,21 +8,13 @@ services: image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro - environment: - # 1 silences the log line of every successful request during load runs. - LB_QUIET: ${LB_QUIET:-0} - # One HAProxy thread per CPU of the limit below. - LB_THREADS: ${LB_THREADS:-1} ports: - # Loopback only; override with PUBLISH=127.0.0.1:28080 when 8080 is busy. - "${PUBLISH:-127.0.0.1:8080}:8080" - # Read-only statistics page: state and checks of every replica. - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" networks: - edge healthcheck: - # Through the balancer itself: healthy once it can route to a ready - # replica, so `compose up --wait` means "ready through the entry point". + # Through the balancer: `up --wait` returns once a replica is reachable. test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] interval: 5s timeout: 3s @@ -38,14 +24,12 @@ services: deploy: resources: limits: - cpus: "${LB_CPUS:-0.5}" + cpus: "0.5" memory: 128M api: build: . image: mtusi-messenger:lab3 - # No container_name and no ports: the service scales, and clients reach - # the replicas only through the balancer. env_file: - path: .env required: false @@ -54,13 +38,8 @@ services: SESSION_STORE_URL_FILE: /run/secrets/session_store_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - # Above the balancer's detection time of 1.5 s (haproxy.cfg). - APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-2s} - DB_MAX_CONNS: ${DB_MAX_CONNS:-10} - # About 90 % of the memory limit below; Go does not read the cgroup limit. - GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} - # Empty disables the internal expvar listener; the benchmark sets :9090. - APP_DEBUG_ADDR: ${APP_DEBUG_ADDR:-} + # Go does not read the cgroup memory limit. + GOMEMLIMIT: 230MiB secrets: - database_url - session_store_url @@ -74,16 +53,15 @@ services: condition: service_healthy valkey: condition: service_healthy - # Drain delay 2 s + shutdown timeout 8 s fit before Docker sends SIGKILL. + # Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL. stop_grace_period: 15s deploy: replicas: ${API_REPLICAS:-2} resources: limits: - cpus: "${API_CPUS:-1.0}" - memory: ${API_MEMORY:-256M} + cpus: "1.0" + memory: 256M - # One-shot: applies the schema once before the replicas start, then exits. migrate: image: mtusi-messenger:lab3 entrypoint: ["/usr/local/bin/migrate"] @@ -108,8 +86,7 @@ services: postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 - # Ten replicas with ten connections each fit during a rolling update of - # five, plus migrate and psql sessions. + # 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql. command: ["postgres", "-c", "max_connections=150"] environment: POSTGRES_DB: messenger @@ -117,28 +94,24 @@ services: POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password secrets: - postgres_password - # Since PostgreSQL 18 the image keeps PGDATA in a versioned directory - # below /var/lib/postgresql, so the volume is mounted on the parent. + # PostgreSQL 18 keeps PGDATA in a versioned directory below this one. volumes: - pgdata:/var/lib/postgresql networks: - data healthcheck: - # Over TCP: during first initialisation the image runs a temporary - # server on the Unix socket only, which must not count as ready. + # Over TCP: the first initialisation runs a temporary socket-only server. test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] interval: 5s timeout: 3s retries: 10 start_period: 10s - # A clean shutdown writes a checkpoint instead of leaving crash recovery - # to the next start. stop_grace_period: 30s deploy: resources: limits: - cpus: "${PG_CPUS:-1.0}" - memory: ${PG_MEMORY:-512M} + cpus: "1.0" + memory: 512M valkey: image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 @@ -165,25 +138,6 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M - # Load generator of the experiments (tools/loadgen). Only - # `docker compose --profile bench run --rm loadgen ...` starts it; it talks to - # the balancer and, for metrics, to the replicas' internal debug port. - loadgen: - profiles: ["bench"] - build: - context: . - target: loadgen - image: mtusi-messenger-loadgen:lab3 - volumes: - - ${BENCH_OUT:-./bench-out}:/out - networks: - - edge - deploy: - resources: - limits: - cpus: "${LOADGEN_CPUS:-1.5}" - memory: 512M - volumes: pgdata: valkeydata: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 861a97d8..7cb20e25 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -1,45 +1,39 @@ # The single entry point: round-robin over the API replicas found through -# Docker DNS, with active readiness checks and no client stickiness. Replicas -# are interchangeable: sessions live in Valkey, everything else in PostgreSQL. +# Docker DNS, with active readiness checks and no client stickiness. global log stdout format raw local0 info maxconn 20000 - # One thread per CPU of the container limit; by default HAProxy would - # start one per CPU of the host and fight the CPU quota. - nbthread "${LB_THREADS}" + # The container has at most one CPU; by default HAProxy starts a thread + # per host CPU and fights the quota. + nbthread 1 defaults mode http log global option dontlognull - # Inside the Docker network a replica accepts or refuses at once; a - # second means it is gone. timeout connect 1s timeout client 30s - # Above the API's own 5 s request deadline: a slow request ends in the - # API's 503 with its cause logged, and this timeout only cuts off a - # replica that hangs entirely. + # Above the API's 5 s request deadline: the API answers a slow request + # itself, this only cuts off a hung replica. timeout server 7s timeout http-request 10s timeout http-keep-alive 10s timeout check 2s - # A request is re-sent, to another replica, only when the connection - # failed and so nothing reached the API. A timeout or an empty response - # may follow a committed write, and POST /users or /chats must never be - # repeated behind the client's back (contract, idempotency section). + # Below the 500 ms latency objective: excess load gets a fast 503. + timeout queue 300ms + # Re-sent only when nothing reached the API: a timeout or an empty + # response may follow a committed write. retries 2 option redispatch 1 retry-on conn-failure - # Errors produced by the balancer itself use the contract's envelope. They - # carry X-Request-Id but no X-Instance-Id: no replica answered. + # Errors produced by the balancer itself use the contract's envelope. http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" -# Docker's embedded DNS: the service name `api` resolves to every running -# replica. Re-resolved every second, so scaled, restarted or replaced -# replicas join and stopped ones leave without a reload. +# `api` resolves to every running replica, re-resolved every second; an +# address that disappears leaves at once. resolvers docker nameserver dns 127.0.0.11:53 resolve_retries 3 @@ -52,10 +46,9 @@ resolvers docker frontend entry bind :8080 option httplog - # 32 hex characters, like the request ids of the API. unique-id-format %[uuid(4),regsub(-,,g)] - # The access log names the replica that answered and its request id, which - # is the key to the replica's own log line. + # Replaces any client value; the API keeps it, so one id spans both logs. + http-request set-header X-Request-Id %[unique-id] capture response header X-Instance-Id len 32 capture response header X-Request-Id len 40 http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 } @@ -66,13 +59,11 @@ backend replicas option httpchk http-check send meth GET uri /health/ready http-check expect status 200 - # Detection: a replica that fails readiness is out after the next check - # (inter 1 s) plus one fast re-check (fastinter 0.5 s), so within 1.5 s. - # A stopping API keeps serving for APP_DRAIN_DELAY (2 s) after readiness - # turns 503, longer than this, and only then closes its listener. Back in - # rotation after two good checks (rise 2), about 2 s. - default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 - # Ten slots: up to five replicas, doubled while docker rollout replaces them. + # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. + # maxconn 20: the replica's 10 pooled database connections plus requests + # that need no database; the rest waits in the queue. 0 means no limit. + default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 + # Up to five replicas, doubled while docker rollout replaces them. server-template api 10 api:8080 listen stats diff --git a/migrations/00002_message_idempotency_keys.sql b/migrations/00002_message_idempotency_keys.sql index 2f35f086..6fe78f6f 100644 --- a/migrations/00002_message_idempotency_keys.sql +++ b/migrations/00002_message_idempotency_keys.sql @@ -1,13 +1,6 @@ -- +goose Up --- One row per Idempotency-Key a user has sent to one chat. The contract scope --- is user + method + chat path + key; only POST /chats/{chat_id}/messages --- accepts keys, so (user_id, chat_id) already fixes the method and the path. --- The primary key is what makes concurrent retries through different --- replicas safe: the second insert of a key waits for the first one's commit. --- The row references the message instead of storing a response: messages --- are immutable, so a replay is answered from the source of truth. --- The table only adds to the schema: the previous API version keeps working --- while a rolling update runs this migration. +-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method + +-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys. CREATE TABLE message_idempotency_keys ( user_id uuid NOT NULL, chat_id uuid NOT NULL, @@ -18,7 +11,6 @@ CREATE TABLE message_idempotency_keys ( FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id) ); --- Serves the periodic removal of keys older than the retention period. CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at); -- +goose Down diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index d7b9d48c..2b8c3115 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -1,5 +1,8 @@ # Shared helpers of the experiment scripts; sourced, not run. +# The generator and the knobs the experiments vary live in compose.bench.yaml. +export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml + # abspath prints the absolute path of an existing directory in the form the # Docker CLI accepts for a bind mount (a Windows path under Git Bash). abspath() { -- 2.54.0 From baa599593817263e34aeb9b7ea95b5dfe74a29b4 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:39:04 +0300 Subject: [PATCH 09/21] =?UTF-8?q?=D0=94=D0=BE=D1=80=D0=B0=D0=B1=D0=BE?= =?UTF-8?q?=D1=82=D0=B0=D1=82=D1=8C=20=D0=B8=D0=BD=D1=81=D1=82=D1=80=D1=83?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=82=D1=8B=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2:=20=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B4=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D1=8B,=20=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=BD=D0=B0=D0=B2=D0=BB=D0=B8=D0=B2=D0=B0=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BB=D0=B5=D1=81=D1=82=D0=BD=D0=B8=D1=86=D1=83=20=D0=BD=D0=B0?= =?UTF-8?q?=20=D0=BF=D0=B5=D1=80=D0=B2=D0=BE=D0=B9=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BB=D0=B5=D0=BD=D0=BD=D0=BE=D0=B9=20=D1=81=D1=82?= =?UTF-8?q?=D1=83=D0=BF=D0=B5=D0=BD=D0=B8,=20=D0=B7=D0=B0=D0=B4=D0=B0?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20maxconn=20=D0=B8=20CPU=20=D0=B1?= =?UTF-8?q?=D0=B0=D0=B7=D1=8B=20=D0=B2=20=D1=81=D0=B5=D1=80=D0=B8=D1=8F?= =?UTF-8?q?=D1=85,=20=D0=B7=D0=B0=D0=BF=D0=B8=D1=81=D1=8B=D0=B2=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=80=20=D1=81=D0=BA?= =?UTF-8?q?=D0=BE=D1=80=D0=BE=D1=81=D1=82=D0=B8=20=D1=85=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B0=20=D0=B8=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=BE=D0=BB?= =?UTF-8?q?=D1=8C=20=D1=88=D1=83=D0=BC=D0=B0,=20=D1=81=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D0=B0=D1=82=D0=B8=D1=82=D1=8C=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 10 +-- scripts/api-instances.sh | 10 +-- scripts/bench/failover.sh | 28 ++----- scripts/bench/lib.sh | 21 ++++-- scripts/bench/plot.py | 74 ++++++++++++------- scripts/bench/rollout-under-load.sh | 22 ++---- scripts/bench/run-series.sh | 109 +++++++++++++++++----------- scripts/bench/series/A.conf | 12 +-- scripts/bench/series/Aprime.conf | 10 +-- scripts/bench/series/B.conf | 12 ++- scripts/bench/series/C.conf | 10 +++ scripts/bench/series/calibrate.conf | 4 - scripts/bench/series/control.conf | 4 + scripts/instances.sh | 4 +- scripts/restart.sh | 7 +- scripts/rollout.sh | 26 ++----- scripts/stop-one.sh | 10 +-- tools/concurrency/client.go | 5 +- tools/concurrency/main.go | 9 +-- tools/loadgen/main.go | 41 ++++------- tools/loadgen/run.go | 11 +-- tools/loadgen/scrape.go | 7 +- tools/loadgen/stats.go | 25 ++++--- tools/loadgen/stats_test.go | 22 ++++++ 24 files changed, 253 insertions(+), 240 deletions(-) create mode 100644 scripts/bench/series/C.conf delete mode 100644 scripts/bench/series/calibrate.conf create mode 100644 scripts/bench/series/control.conf diff --git a/Makefile b/Makefile index 39794250..21b7346d 100644 --- a/Makefile +++ b/Makefile @@ -67,17 +67,15 @@ logs: instances: sh scripts/instances.sh -# Replaces every API replica with one built from the current sources, without -# downtime; restore a replica stopped by stop-one with `docker compose start api`. +# Replaces every API replica without downtime; all of them must be running. rollout: sh scripts/rollout.sh -# Track 1, the cost of scaling: one series on fresh stands, then the charts. -# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800 +# Track 1. Every run starts from empty volumes, so the stand's data is lost. +# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200 SERIES ?= scripts/bench/series/A.conf -REPEATS ?= 3 bench: - sh scripts/bench/run-series.sh $(SERIES) $(REPEATS) + sh scripts/bench/run-series.sh $(SERIES) # Needs matplotlib from requirements-bench.txt: make plot PYTHON=. plot: diff --git a/scripts/api-instances.sh b/scripts/api-instances.sh index 4a90d4f1..be6ce307 100755 --- a/scripts/api-instances.sh +++ b/scripts/api-instances.sh @@ -1,11 +1,7 @@ #!/bin/sh -# Lists the API containers of this compose project, one per line: -# -# The alias is what the process sends as X-Instance-Id. It is random per -# process, so it is asked from the process itself, over the same wget the -# Docker healthcheck uses, from inside the container. A container that is not -# running has no alias and shows "-". The replicas are asked in parallel: the -# failover hook that relies on this list must return within 10 seconds. +# Lists the API containers: . The +# alias is random per process, so each running replica is asked over +# /health/live, in parallel: the failover hook must return within 10 seconds. set -eu cd "$(dirname "$0")/.." diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh index d223fc21..afff54d9 100755 --- a/scripts/bench/failover.sh +++ b/scripts/bench/failover.sh @@ -1,26 +1,14 @@ #!/bin/sh -# Level 4: one API replica fails under a constant write load. The generator -# sends messages at RATE for DURATION and retries uncertain sends with the -# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM, -# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started -# again. Afterwards the generator reads the history back: every confirmed -# message once. +# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and +# started again at RESTORE_AT, under a constant write load that retries +# uncertain sends with the same key; then every confirmed message must be in +# the history once. # # sh scripts/bench/failover.sh stop|kill # -# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), -# OUT (default evidence/failover). Results in /-/: -# timeline.json, seconds.csv (per-second outcomes, attributed by send time), -# faults.json (when and what was stopped), compose.log (the balancer's -# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. -# Runs on the current stand (`make up`) and leaves every replica running. -# -# Every time used for the analysis comes from the Docker VM's clock: the log -# timestamps, the container's stop and start times from `docker inspect`, and -# the generator's own times. The host clock of Windows runs apart from it by -# a fraction of a second, a large share of a 1.5 s detection window; host -# times are kept for reference only. With `stop`, the moment SIGTERM arrived -# is the replica's own "shutdown started" line in compose.log. +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT +# (evidence/failover). Times come from the Docker VM clock; the Windows clock +# differs by a fraction of a second and is kept for reference only. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -50,8 +38,6 @@ since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! -# The generator seeds two users first; its timeline starts after that. The -# fault times below are absolute, as is started_at in timeline.json. sleep "$fault_at" fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker "$kind" "$target" >/dev/null diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index 2b8c3115..24232ee8 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -3,14 +3,13 @@ # The generator and the knobs the experiments vary live in compose.bench.yaml. export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml -# abspath prints the absolute path of an existing directory in the form the -# Docker CLI accepts for a bind mount (a Windows path under Git Bash). +# abspath prints a directory in the form the Docker CLI mounts (a Windows path +# under Git Bash). abspath() { (cd "$1" && (pwd -W 2>/dev/null || pwd)) } -# meta writes the header of one run: what ran, when, on which commit and -# host. Arguments: file, then key=value pairs added to the record. +# meta writes what ran, when, on which commit and host: file, then key=value. meta() { file=$1 shift @@ -30,8 +29,13 @@ meta() { } >"$file" } -# sample_docker_stats appends the CPU and memory of every container of the -# stand to a JSON-lines file, about every two seconds, until killed. +# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: +# the same work every time, so a slow spell of the host shows in the results. +host_probe() { + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' +} + +# sample_docker_stats appends the CPU and memory of every container until killed. sample_docker_stats() { while :; do docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ @@ -39,8 +43,8 @@ sample_docker_stats() { done >>"$1" } -# sample_postgres appends, every second, the database's commit counters and -# what its active sessions are waiting for (none: running on CPU). +# sample_postgres appends, every second, the database counters and what the +# active sessions wait for ("cpu": nothing). sample_postgres() { while :; do docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " @@ -48,6 +52,7 @@ sample_postgres() { 't', extract(epoch FROM now()), 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'sessions', sessions, 'sessions_abandoned', sessions_abandoned, 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index 20493b2d..33305612 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -5,14 +5,16 @@ The raw root holds //rep/ directories. Written to the output directory: - summary.csv one row per step of every run: offered and successful rate, - errors, latency percentiles, generator lag, mean CPU of each - component and pool waits during the step; + summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; + control.csv the noise checks of every run: the host speed probe and the + loadgen CPU per request at the first step, flagged when more + than 25 % off the median; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png. -Capacity is the highest valid step (generator on schedule) with less than 1 % -errors and a p99 of successful requests within 500 ms. +Capacity is the highest step that passed the criterion loadgen applies (valid, +under 1 % errors, p99 of successful requests within 500 ms). Charts show the +median of the repeats; a hollow marker means the generator fell behind. """ import csv import json @@ -27,7 +29,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 SLO_P99_MS = 500.0 -MAX_ERROR_SHARE = 0.01 +CONTROL_TOLERANCE = 0.25 COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") @@ -69,6 +71,7 @@ def load_runs(root): "steps": report["steps"], "docker": read_jsonl(d / "docker-stats.jsonl"), "metrics": read_jsonl(d / "metrics.jsonl"), + "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, }) return runs @@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1): return out -def pool_waits(run, t0, t1): - """Requests that waited for a pooled connection during [t0, t1), all - replicas together, per second.""" +def pool_rate(run, t0, t1, counter): + """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" first, last = {}, {} for row in run["metrics"]: t = run["start"] + row["t"] @@ -104,18 +106,14 @@ def pool_waits(run, t0, t1): for replica in row.get("replicas") or []: pool = replica.get("pgxpool") or {} key = replica.get("instance") - if key and "empty_acquire_count" in pool: - first.setdefault(key, pool["empty_acquire_count"]) - last[key] = pool["empty_acquire_count"] + if key and counter in pool: + first.setdefault(key, pool[counter]) + last[key] = pool[counter] if not last: return "" return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) -def is_capacity(step): - return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS - - def main(root, out): runs = load_runs(root) if not runs: @@ -129,17 +127,22 @@ def main(root, out): t0 = run["start"] + step["started_at_s"] t1 = t0 + step["seconds"] cpu = mean_cpu(run, t0, t1) + classes = step["classes"] row = { "series": run["series"], "config": run["config"], "rep": run["rep"], "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], - "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], + "client_timeouts": classes.get("client_timeout", 0), + "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), + "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, - "pool_waits_per_s": pool_waits(run, t0, t1), + "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), + "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), } rows.append(row) - if is_capacity(step): + if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): best = row capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], @@ -164,13 +167,29 @@ def main(root, out): medians[key] = statistics.median(values) w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + control = [] + for run in runs: + first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] + if first and first[0]["cpu_loadgen_pct"] != "": + control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"], + round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)]) + if control: + median = statistics.median(c[5] for c in control) + with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"]) + for c in control: + deviation = c[5] / median - 1 + w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE]) + def median_by_offered(series, config, field): - points = {} + points, valid = {}, {} for r in rows: - if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "": + if r["series"] == series and r["config"] == config and r[field] != "": points.setdefault(r["offered_rps"], []).append(float(r[field])) + valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] xs = sorted(points) - return xs, [statistics.median(points[x]) for x in xs] + return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] for series in sorted({s for s, _ in configs}): names = [c for s, c in configs if s == series] @@ -181,8 +200,11 @@ def main(root, out): fig, ax = plt.subplots(figsize=(7, 4.5)) top = 0 for name in names: - xs, ys = median_by_offered(series, name, field) - ax.plot(xs, ys, marker="o", label=name) + xs, ys, valid = median_by_offered(series, name, field) + line, = ax.plot(xs, ys, label=name) + ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) + ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], + facecolors="none", edgecolors=line.get_color()) top = max([top, *xs]) if field == "ok_rps" and top: ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") @@ -192,7 +214,7 @@ def main(root, out): ax.set_yscale("log") ax.set_xlabel("Offered requests per second") ax.set_ylabel(title) - ax.set_title(f"Series {series}: median of repeats, valid steps only") + ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() @@ -233,7 +255,7 @@ def main(root, out): fig, ax = plt.subplots(figsize=(7, 4.5)) for series, name in configs: - xs, ys = median_by_offered(series, name, "pool_waits_per_s") + xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") if xs: ax.plot(xs, ys, marker="o", label=f"{series}/{name}") ax.set_xlabel("Offered requests per second") diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh index 2390155a..44cfb40b 100755 --- a/scripts/bench/rollout-under-load.sh +++ b/scripts/bench/rollout-under-load.sh @@ -1,21 +1,12 @@ #!/bin/sh -# Level 4: a rolling update under a constant write load. The generator sends -# messages at RATE for DURATION, retrying uncertain sends with the same key; -# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards -# the generator reads the history back: every confirmed message once. +# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a +# constant write load that retries uncertain sends with the same key; then +# every confirmed message must be in the history once. # # sh scripts/bench/rollout-under-load.sh # -# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default -# evidence/rollout). Results in /: timeline.json, seconds.csv, -# rollout.log, update.json (when it ran, which containers it replaced), -# compose.log, instances-*.txt. -# -# Times for the analysis come from the Docker VM's clock, as the generator's -# do: log timestamps and the start times of the new containers. The host -# clock of Windows runs apart from it and is kept for reference only. The -# old replicas are removed by the update; their stop shows in compose.log as -# their "shutdown started" lines. +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout). +# Times come from the Docker VM clock, as in failover.sh. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -29,8 +20,7 @@ export MSYS_NO_PATHCONV=1 rm -rf "$dir" mkdir -p "$dir" docker compose up -d --wait >/dev/null 2>&1 -# Build first, as a delivery pipeline would: the update under load then -# measures the replacement of the replicas, not the compiler. +# Built beforehand, as by a delivery pipeline: the update replaces replicas. docker compose build api >/dev/null 2>&1 meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" sh scripts/instances.sh >"$dir/instances-before.txt" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index 11d98aa8..b25d7389 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -1,61 +1,82 @@ #!/bin/sh -# Track 1, the cost of scaling: runs every configuration of one series file, -# each repeated, each on a fresh stand (empty volumes, same seed), under the -# same grid of offered rates. +# Track 1: every configuration of the given series files, each on a fresh +# stand, under one grid of offered rates. Repeats are interleaved (repeat 1 +# of every configuration, then repeat 2), so a slow spell of the host does not +# fall on one configuration. # -# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats] +# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...] # -# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root, -# default evidence/track1/raw). Fixed for every configuration: HAProxy -# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the -# generator LOADGEN_CPUS (1.5). Results per run: -# ///rep/{meta.json,steps.json,seconds.csv, -# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log} -# The stand is removed at the end; `make up` brings the normal one back. +# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s), +# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT +# (evidence/track1/raw). Results: ///rep/. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh -series=${1:?usage: run-series.sh [repeats]} -repeats=${2:-3} -: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}" +[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh ..." >&2; exit 2; } +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}" +repeats=${REPEATS:-3} step=${STEP:-30s} warmup=${WARMUP:-30s} -root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf) +out=${OUT:-evidence/track1/raw} +stop= +[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail -export APP_DEBUG_ADDR=:9090 LB_QUIET=1 -export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5} +# Fixed for every configuration; the API, its pool, maxconn and the database +# CPU come from the series line. +export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 export MSYS_NO_PATHCONV=1 -grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do - [ -n "$name" ] || continue - for rep in $(seq 1 "$repeats"); do - dir=$root/$name/rep$rep - rm -rf "$dir" - mkdir -p "$dir" - echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES" - export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool +# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus. +run_config() { + series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10} + dir=$out/$series/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \ + DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus - docker compose down -v --remove-orphans >/dev/null 2>&1 - docker compose up -d --build --wait >/dev/null 2>&1 - meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \ - "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \ - "lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \ - "rates=$RATES" "step=$step" "warmup=$warmup" + docker compose down -v --remove-orphans >/dev/null 2>&1 + if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then + echo " the stand did not start, see $dir/up.log" >&2 + exit 1 + fi + meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \ + "db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \ + "lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \ + "stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)" - sample_docker_stats "$dir/docker-stats.jsonl" & - stats=$! - sample_postgres "$dir/postgres.jsonl" & - pg=$! - # stdin is closed: `compose run` is interactive by default and would - # otherwise read the rest of the series file this loop is reading. - BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ - -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 /dev/null || true - wait "$stats" "$pg" 2>/dev/null || true - cat "$dir/loadgen.log" | grep '^step' || true + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + # stdin closed: `compose run` would otherwise read it. + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 /dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1 + grep '^step' "$dir/loadgen.log" || true +} + +docker compose --profile bench build loadgen >/dev/null +default_ifs=$IFS +newline=' +' +for rep in $(seq 1 "$repeats"); do + for conf in "$@"; do + lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d') + IFS=$newline + for line in $lines; do + IFS=$default_ifs + # shellcheck disable=SC2086 # the line splits into the fields + run_config "$(basename "$conf" .conf)" "$rep" $line + done + IFS=$default_ifs done done docker compose down -v --remove-orphans >/dev/null 2>&1 -echo "results in $root" +echo "results in $out" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf index 1a305380..a461f2b3 100644 --- a/scripts/bench/series/A.conf +++ b/scripts/bench/series/A.conf @@ -1,6 +1,6 @@ -# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas. -# The pool stays 10 per replica, so the total grows with the replicas. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -A1 1 2.0 1024M 920MiB 10 -A2 2 1.0 512M 460MiB 10 -A4 4 0.5 256M 230MiB 10 +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas; +# the pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A1 1 2.0 1024M 920MiB 10 0 1.0 +A2 2 1.0 512M 460MiB 10 0 1.0 +A4 4 0.5 256M 230MiB 10 0 1.0 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf index d0952eff..6ebe2902 100644 --- a/scripts/bench/series/Aprime.conf +++ b/scripts/bench/series/Aprime.conf @@ -1,5 +1,5 @@ -# Series A': the budget of series A with the total pool fixed at 20 -# connections. A'2 equals A2 and is not run again. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -Ap1 1 2.0 1024M 920MiB 20 -Ap4 4 0.5 256M 230MiB 5 +# Series A': the budget of series A with the total pool fixed at 20 connections; +# A'2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +Ap1 1 2.0 1024M 920MiB 20 0 1.0 +Ap4 4 0.5 256M 230MiB 5 0 1.0 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf index b882d44e..9f4fc11c 100644 --- a/scripts/bench/series/B.conf +++ b/scripts/bench/series/B.conf @@ -1,7 +1,5 @@ -# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by -# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run -# again. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -B1 1 1.0 512M 460MiB 10 -B3 3 1.0 512M 460MiB 10 -B4 4 1.0 512M 460MiB 10 +# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +B1 1 1.0 512M 460MiB 10 0 1.0 +B3 3 1.0 512M 460MiB 10 0 1.0 +B4 4 1.0 512M 460MiB 10 0 1.0 diff --git a/scripts/bench/series/C.conf b/scripts/bench/series/C.conf new file mode 100644 index 00000000..20d012f3 --- /dev/null +++ b/scripts/bench/series/C.conf @@ -0,0 +1,10 @@ +# Improvement C: maxconn per replica on the balancer, before (0, no limit) and +# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0); +# 10 and 40 show how much the value matters. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +R1m0 1 1.0 512M 460MiB 10 0 1.0 +R1m20 1 1.0 512M 460MiB 10 20 1.0 +R4m0 4 1.0 512M 460MiB 10 0 1.0 +R4m20 4 1.0 512M 460MiB 10 20 1.0 +R1m10 1 1.0 512M 460MiB 10 10 1.0 +R1m40 1 1.0 512M 460MiB 10 40 1.0 diff --git a/scripts/bench/series/calibrate.conf b/scripts/bench/series/calibrate.conf deleted file mode 100644 index 0e3c976e..00000000 --- a/scripts/bench/series/calibrate.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Calibration: the smallest API budget of the study (one replica, one CPU) -# with a wide range of rates finds where the common rate grid must reach. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -cal 1 1.0 512M 460MiB 10 diff --git a/scripts/bench/series/control.conf b/scripts/bench/series/control.conf new file mode 100644 index 00000000..d00c775c --- /dev/null +++ b/scripts/bench/series/control.conf @@ -0,0 +1,4 @@ +# Control: A2 with twice the database CPU. If the database is the bottleneck, +# the capacity rises. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A2pg2 2 1.0 512M 460MiB 10 0 2.0 diff --git a/scripts/instances.sh b/scripts/instances.sh index 52150a3f..02fbbefd 100755 --- a/scripts/instances.sh +++ b/scripts/instances.sh @@ -1,7 +1,5 @@ #!/bin/sh -# Shows which container answers under which X-Instance-Id: the proof on the -# defense that the header values belong to different running processes, and -# which container a failover test actually stopped. +# Which container answers under which X-Instance-Id. set -eu cd "$(dirname "$0")/.." diff --git a/scripts/restart.sh b/scripts/restart.sh index 29aeb757..7eda3b6a 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,9 +1,6 @@ #!/bin/sh -# Restart hook of `make test-persistence` (tests/resilience.py): stops the -# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the -# same containers again (the one-shot migrate finds nothing to apply). -# Volumes and data are kept and nothing is re-seeded. The test hides the -# output, so failures surface through the exit code only. +# Restart hook of `make test-persistence`: stops the whole stand gracefully and +# starts the same containers again; volumes are kept. set -eu cd "$(dirname "$0")/.." docker compose stop diff --git a/scripts/rollout.sh b/scripts/rollout.sh index c46e52c4..4710e64c 100755 --- a/scripts/rollout.sh +++ b/scripts/rollout.sh @@ -1,21 +1,12 @@ #!/bin/sh -# Rolling update of the API without downtime, with docker rollout -# (https://github.com/wowu/docker-rollout, MIT): it starts as many new -# replicas as are running, waits until all of them are healthy, then stops -# and removes the old ones. If a new replica does not become healthy it -# removes the new ones and leaves the old ones serving, so the stand is never -# left with a mix of versions. +# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout, +# MIT): new replicas start next to the old ones, and the old ones go once all +# new ones are healthy; if one is not, the new ones are removed instead. # -# Every replica must be running. With a stopped one (say, after stop-one), -# docker rollout only replaces the stopped one and reports success, which -# would leave old and new versions side by side; the script refuses instead. -# -# The migrations of the new version run first, once, while the old replicas -# serve: they only add to the schema. docker rollout scales with `compose up`, -# which also starts the dependencies of the API and would rerun the old, -# exited migrate container. Recreating it beforehand from the new image, -# without starting it, makes that `up` run the new migrations exactly once, -# before any new replica starts. +# With a stopped replica docker rollout would replace only that one and leave +# two versions running, so the script refuses. The migrate container is +# recreated from the new image without starting: the `up` of docker rollout +# then applies the new migrations once, before any new replica starts. set -eu cd "$(dirname "$0")/.." @@ -44,8 +35,7 @@ docker compose build api docker compose up --no-start --no-deps --force-recreate migrate # Run through sh: Windows cannot use the script as a docker CLI plugin. if ! sh "$tool" rollout --wait-after-healthy 4 api; then - # A failed migration stops the scale-up after the new containers were - # created but before they started; remove them for a clean next attempt. + # A failed migration leaves the new containers created but not started. created=$(docker compose ps -aq --status created api) if [ -n "$created" ]; then docker rm $created >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index d2730e94..6971e1b4 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -1,11 +1,7 @@ #!/bin/sh -# Failover hook of `make test-failover` (tests/resilience.py): stops the one -# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it -# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps -# serving for its drain delay while HAProxy takes it out of rotation, then -# exits; `docker stop` returns once it has. Nothing else is touched, and an -# alias that does not name exactly one running replica stops nothing. -# Restore the replica with: docker compose start api +# Failover hook of `make test-failover`: gracefully stops the one running API +# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else. +# Restore it with: docker compose start api set -eu cd "$(dirname "$0")/.." diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go index b56eb137..9b583e0d 100644 --- a/tools/concurrency/client.go +++ b/tools/concurrency/client.go @@ -52,9 +52,8 @@ func (c *client) postMessage(chatID, token, text string, out any) error { return c.do(req, token, http.StatusCreated, out) } -// sendWithKey sends one attempt of a message under the given key on a -// connection of its own and reports the status, the message (on 201) and the -// replica that answered. +// sendWithKey sends one attempt on a connection of its own, so that the +// attempts of one key reach different replicas. func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { var msg chat.MessageResponse raw, err := json.Marshal(map[string]string{"text": text}) diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go index 7011c071..a1ac25e6 100644 --- a/tools/concurrency/main.go +++ b/tools/concurrency/main.go @@ -4,12 +4,9 @@ // then the history is paged and must contain exactly N*M unique messages in // id order. // -// Phase 2, shared idempotency: in each of K rounds one user sends the same -// message with the same Idempotency-Key P times in parallel, each attempt on -// its own connection, so the balancer spreads them over the replicas. Every -// attempt must answer 201 with the same message, the attempts must have -// reached at least two replicas, the history must hold exactly K messages, -// and the key with another text must answer 409. +// Phase 2: in each of K rounds one key is sent P times in parallel through the +// balancer; every attempt must get 201 with one message, at least two replicas +// must answer, the history must hold K messages, and another text must get 409. package main import ( diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go index fabeddd7..e9dd9fc0 100644 --- a/tools/loadgen/main.go +++ b/tools/loadgen/main.go @@ -1,21 +1,10 @@ -// Command loadgen is the open-loop load generator of the scaling study -// (track 1) and of the failover and rolling-update experiments. +// Command loadgen is the open-loop load generator of the experiments. // -// loadgen bench — seeds users, chats and messages, warms up, then offers -// each rate of -rates for -step: a mix of history reads -// (-read-ratio) and message sends. Writes steps.json and -// seconds.csv, and metrics.jsonl with -scrape. -// loadgen timeline — one user sends messages into one chat at -rate for -// -duration; a send with an uncertain outcome (no answer, -// 5xx) is retried with the same Idempotency-Key. At the -// end the whole history is read back: every confirmed -// message must be there, and none twice. Writes -// timeline.json and seconds.csv. -// -// Requests are sent on schedule whatever the response times (open loop), and -// the generator records how late it sent them: a step it could not keep up -// with is marked invalid rather than reported as the system's limit. Tokens -// stay in memory and are never written out. +// loadgen bench — seeds data, warms up, then offers each rate of -rates +// for -step as a mix of history reads and message sends. +// loadgen timeline — one user sends messages at -rate, retrying uncertain +// sends with the same Idempotency-Key, then reads the +// history back: every confirmed message once. package main import ( @@ -52,6 +41,7 @@ type config struct { Timeout string `json:"client_timeout"` Seed uint64 `json:"seed"` Scrape bool `json:"scrape"` + StopOnFail bool `json:"stop_on_fail"` StatsURL string `json:"stats_url"` APIHost string `json:"api_host"` DebugPort string `json:"debug_port"` @@ -98,6 +88,7 @@ func parse(mode string, args []string) (config, error) { fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion") fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") @@ -138,8 +129,7 @@ func writeJSON(path string, v any) error { return os.WriteFile(path, append(raw, '\n'), 0o644) } -// startScrape samples the system in the background when asked to; the -// returned function stops it and waits. +// startScrape returns the function that stops the sampling and waits. func startScrape(ctx context.Context, cfg config, start time.Time) func() { if !cfg.Scrape { return func() {} @@ -177,8 +167,7 @@ func runBench(ctx context.Context, cfg config) error { stopScrape := startScrape(ctx, cfg, start) defer stopScrape() - // next picks the request on the scheduling goroutine, where the random - // source may be used safely. + // next runs on the scheduling goroutine, which owns the random source. counter := 0 next := func(rec *recorder) func(time.Time) func() { return func(planned time.Time) func() { @@ -217,8 +206,11 @@ func runBench(ctx context.Context, cfg config) error { for _, x := range rec.all() { all.add(x) } - fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n", - rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid) + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) + if cfg.StopOnFail && !s.Passed { + break + } } stopScrape() if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { @@ -247,8 +239,7 @@ type timelineReport struct { // retryDelay spaces the attempts of one message whose outcome was uncertain. const retryDelay = 200 * time.Millisecond -// retryGrace is how long after the schedule ends an uncertain message keeps -// being retried before it counts as unresolved. +// retryGrace bounds the retries of an uncertain message after the schedule ends. const retryGrace = 10 * time.Second func runTimeline(ctx context.Context, cfg config) error { diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go index 21019fd3..7a84f6a2 100644 --- a/tools/loadgen/run.go +++ b/tools/loadgen/run.go @@ -6,14 +6,9 @@ import ( "time" ) -// openLoop offers requests at a constant rate for d, whatever the response -// times: request i is planned at start + i/rate and sent then, or at once if -// the generator is already late. Waiting for responses before sending more -// (a closed loop) would let a slow system lower its own load and hide its -// latency: coordinated omission. next runs on the scheduling goroutine, so it -// may use non-thread-safe state such as the random source; the job it -// returns runs on a goroutine of its own. openLoop returns once every job -// has finished. +// openLoop sends request i at start + i/rate whatever the response times: a +// closed loop would let a slow system lower its own load and hide its latency +// (coordinated omission). It returns once every job has finished. func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { n := int(rate * d.Seconds()) interval := time.Duration(float64(time.Second) / rate) diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go index 69790f23..6b95db75 100644 --- a/tools/loadgen/scrape.go +++ b/tools/loadgen/scrape.go @@ -12,11 +12,8 @@ import ( "time" ) -// scraper samples, once a second, what the balancer and every replica report -// about themselves, so a saturated step can be explained: the HAProxy -// statistics per replica and the expvar of each API process (pool waits, -// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them, -// through Docker DNS, and asked directly on their internal debug port. +// scraper samples the HAProxy statistics and every replica's expvar once a +// second; replicas are found through Docker DNS, as HAProxy finds them. type scraper struct { statsURL string apiHost string diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go index f74336e9..541299f0 100644 --- a/tools/loadgen/stats.go +++ b/tools/loadgen/stats.go @@ -12,11 +12,17 @@ import ( "time" ) -// maxValidLag is the generator's own tolerance: if requests left more than -// this late at the 99th percentile, the generator, not the system, limited -// the step, and the step is excluded from the conclusions. +// maxValidLag: if requests left later than this at p99, the generator, not +// the system, limited the step. const maxValidLag = 10 * time.Millisecond +// The capacity criterion of the study: a step passes when it is valid, has +// fewer than 1 % errors and its successful requests have a p99 within 500 ms. +const ( + maxErrorShare = 0.01 + sloP99 = 500 * time.Millisecond +) + // sample is one request as the generator saw it. type sample struct { planned time.Time @@ -27,11 +33,9 @@ type sample struct { write bool } -// classify names the outcome of a request. HTTP errors produced by the -// balancer itself carry no X-Instance-Id and are kept apart from the API's -// own: a 503 from HAProxy means no replica was available, a 503 from the API -// means a dependency was. client_timeout is the generator giving up on a -// response, not a timeout of the system. +// classify names the outcome of a request. Errors of the balancer itself carry +// no X-Instance-Id and get the suffix _lb; client_timeout is the generator +// giving up, not a timeout of the system. func classify(r result) string { switch { case r.err != nil && isTimeout(r.err): @@ -78,12 +82,12 @@ type stepSummary struct { P99Ms float64 `json:"ok_p99_ms"` LagP99Ms float64 `json:"lag_p99_ms"` Valid bool `json:"valid"` + Passed bool `json:"passed"` Instances map[string]int `json:"instances"` Writes int `json:"writes_ok"` } -// summarize reduces one step. Latency percentiles cover successful requests -// only, as the course's report format asks; the errors are counted by class. +// summarize reduces one step; latency percentiles cover successful requests. func summarize(rate float64, d time.Duration, samples []sample) stepSummary { s := stepSummary{ OfferedRPS: rate, @@ -117,6 +121,7 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary { lagP99 := percentile(lags, 99) s.LagP99Ms = ms(lagP99) s.Valid = lagP99 <= maxValidLag + s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99 return s } diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index 1e590016..df7b6fea 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -104,3 +104,25 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { t.Fatalf("open loop took %v, want about 1.2 s", took) } } + +func TestStepPassesOnlyWithinTheObjective(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, fast); !s.Passed { + t.Fatalf("fast clean step must pass: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); s.Passed { + t.Fatalf("p99 above 500 ms must fail: %+v", s) + } + failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond}) + if s := summarize(3, time.Second, failing); s.Passed { + t.Fatalf("errors above 1 %% must fail: %+v", s) + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.Passed { + t.Fatalf("an invalid step must fail: %+v", s) + } +} -- 2.54.0 From f5f8040783c5c2f30b6e430b09fa05a81c5e655e Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:58:00 +0300 Subject: [PATCH 10/21] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E:=20=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=BD=D0=B0=D0=B2=D0=BB=D0=B8=D0=B2=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BB=D0=B5=D1=81=D1=82=D0=BD=D0=B8=D1=86=D1=83=20?= =?UTF-8?q?=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=BF=D1=80=D0=B8=20?= =?UTF-8?q?=D0=BF=D0=B5=D1=80=D0=B5=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B5=20?= =?UTF-8?q?=D1=81=D0=B8=D1=81=D1=82=D0=B5=D0=BC=D1=8B,=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B5=D1=80=D1=8F=D1=82=D1=8C=20=D1=84=D0=B0=D0=B9?= =?UTF-8?q?=D0=BB=D1=8B=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9=20=D0=B4=D0=BE=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA=D0=B0,=20=D0=B8=D1=81?= =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20=D0=B5=D0=B4=D0=B8?= =?UTF-8?q?=D0=BD=D0=B8=D1=86=D1=8B=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=BE?= =?UTF-8?q?=D0=BB=D1=8F=20=D1=88=D1=83=D0=BC=D0=B0,=20=D1=83=D0=B1=D1=80?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BB=D0=B8=D1=88=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B8?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra/haproxy/haproxy.cfg | 2 +- internal/platform/config/config.go | 5 +---- scripts/bench/lib.sh | 3 ++- scripts/bench/plot.py | 35 +++++++++++++++++++----------- scripts/bench/run-series.sh | 5 +++++ tools/loadgen/main.go | 4 ++-- tools/loadgen/stats.go | 6 ++++- tools/loadgen/stats_test.go | 20 +++++++++++++++++ 8 files changed, 58 insertions(+), 22 deletions(-) diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 7cb20e25..129badc3 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -61,7 +61,7 @@ backend replicas http-check expect status 200 # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. # maxconn 20: the replica's 10 pooled database connections plus requests - # that need no database; the rest waits in the queue. 0 means no limit. + # that need no database; the rest waits in the queue. default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 # Up to five replicas, doubled while docker rollout replaces them. server-template api 10 api:8080 diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 49e191f3..936b8ddc 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -15,8 +15,6 @@ import ( "time" ) -// Logging selects the log format, text for dev and JSON otherwise, and the -// level. type Logging struct { Env string Level slog.Level @@ -44,8 +42,7 @@ type Config struct { DrainDelay time.Duration IdempotencyTTL time.Duration DBMaxConns int32 - // DebugAddr enables the internal expvar listener when set. - DebugAddr string + DebugAddr string } type Migrate struct { diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index 24232ee8..991fe07c 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -32,7 +32,8 @@ meta() { # host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: # the same work every time, so a slow spell of the host shows in the results. host_probe() { - docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 \ + -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' } # sample_docker_stats appends the CPU and memory of every container until killed. diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index 33305612..cced9925 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -7,9 +7,9 @@ The raw root holds //rep/ directories. Written to the output directory: summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; - control.csv the noise checks of every run: the host speed probe and the - loadgen CPU per request at the first step, flagged when more - than 25 % off the median; + control.csv the noise checks of every run, the host speed probe and the + loadgen CPU per request at the first step, each flagged when + more than 25 % off its median; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png. Capacity is the highest step that passed the criterion loadgen applies (valid, @@ -147,6 +147,7 @@ def main(root, out): capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], "capacity_rps": best["offered_rps"] if best else 0, "at": best, + "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), }) with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: @@ -160,27 +161,35 @@ def main(root, out): configs.append((c["series"], c["config"])) with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) - w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median"]) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) medians = {} for key in configs: - values = [c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == key] + mine = [c for c in capacities if (c["series"], c["config"]) == key] + values = [c["capacity_rps"] for c in mine] medians[key] = statistics.median(values) - w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", + sum(c["invalid_below"] for c in mine)]) control = [] for run in runs: first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] if first and first[0]["cpu_loadgen_pct"] != "": - control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"], - round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)]) + probe = run["meta"].get("host_probe_s", "") + control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, + round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) if control: - median = statistics.median(c[5] for c in control) + probes = [c[3] for c in control if c[3] is not None] + probe_median = statistics.median(probes) if probes else None + cpu_median = statistics.median(c[4] for c in control) with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) - w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"]) - for c in control: - deviation = c[5] / median - 1 - w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE]) + w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", + "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) + for series, config, rep, probe, cpu in control: + probe_dev = probe / probe_median - 1 if probe is not None else 0.0 + cpu_dev = cpu / cpu_median - 1 + w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, + f"{cpu_dev:+.2f}", max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE]) def median_by_offered(series, config, field): points, valid = {}, {} diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index b25d7389..219665f0 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -62,6 +62,11 @@ run_config() { grep '^step' "$dir/loadgen.log" || true } +for conf in "$@"; do + [ -r "$conf" ] || { echo "cannot read $conf" >&2; exit 2; } + bad=$(grep -v '^#' "$conf" | awk 'NF && NF != 8') + [ -z "$bad" ] || { echo "$conf: a line needs 8 fields: $bad" >&2; exit 2; } +done docker compose --profile bench build loadgen >/dev/null default_ifs=$IFS newline=' diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go index e9dd9fc0..f081d981 100644 --- a/tools/loadgen/main.go +++ b/tools/loadgen/main.go @@ -88,7 +88,7 @@ func parse(mode string, args []string) (config, error) { fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") - fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99") fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") @@ -208,7 +208,7 @@ func runBench(ctx context.Context, cfg config) error { } fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) - if cfg.StopOnFail && !s.Passed { + if cfg.StopOnFail && s.overloaded() { break } } diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go index 541299f0..8f5dd8ca 100644 --- a/tools/loadgen/stats.go +++ b/tools/loadgen/stats.go @@ -121,10 +121,14 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary { lagP99 := percentile(lags, 99) s.LagP99Ms = ms(lagP99) s.Valid = lagP99 <= maxValidLag - s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99 + s.Passed = s.Valid && !s.overloaded() return s } +func (s stepSummary) overloaded() bool { + return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99) +} + func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index df7b6fea..f9a8bf1b 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -126,3 +126,23 @@ func TestStepPassesOnlyWithinTheObjective(t *testing.T) { t.Fatalf("an invalid step must fail: %+v", s) } } + +// The ladder stops when the system leaves the objective, not when the +// generator alone fell behind: a later step may still be valid and pass. +func TestOnlyTheSystemEndsTheLadder(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.overloaded() { + t.Fatalf("a lagging generator must not end the ladder: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); !s.overloaded() { + t.Fatalf("p99 above 500 ms must end the ladder: %+v", s) + } + failing := []sample{ok(10*time.Millisecond, time.Millisecond), {class: "client_timeout", lag: time.Millisecond}} + if s := summarize(2, time.Second, failing); !s.overloaded() { + t.Fatalf("errors above 1 %% must end the ladder: %+v", s) + } +} -- 2.54.0 From 1626145830bc739186abf024d4c2863e4989aeaf Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 13:09:44 +0300 Subject: [PATCH 11/21] =?UTF-8?q?=D0=9D=D0=B5=20=D0=BF=D0=B8=D1=81=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B2=20=D0=BB=D0=BE=D0=B3=20=D0=B1=D0=B0=D0=BB?= =?UTF-8?q?=D0=B0=D0=BD=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=20=D1=83=D1=81=D0=BF=D0=B5=D1=88=D0=BD=D1=8B=D0=B5=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D1=8B=20=D0=B2=D0=BE=20?= =?UTF-8?q?=D0=B2=D1=81=D0=B5=D1=85=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/bench/lib.sh | 2 ++ scripts/bench/run-series.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index 991fe07c..df5e37fc 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -2,6 +2,8 @@ # The generator and the knobs the experiments vary live in compose.bench.yaml. export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml +# The balancer logs errors and state changes only, not every request. +export LB_QUIET=1 # abspath prints a directory in the form the Docker CLI mounts (a Windows path # under Git Bash). diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index 219665f0..f8b8de71 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -24,7 +24,7 @@ stop= # Fixed for every configuration; the API, its pool, maxconn and the database # CPU come from the series line. -export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 +export LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 export MSYS_NO_PATHCONV=1 # One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus. -- 2.54.0 From 0b99894fefc4c2eef2d2ae4a46786d975b205f15 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 13:16:03 +0300 Subject: [PATCH 12/21] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B4=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=81=D1=82=D0=B2=D0=B0=20=D1=83=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=BD=D0=B5=D0=B9=203=20=D0=B8=204:=20=D0=B7=D0=B0=D0=BF=D1=83?= =?UTF-8?q?=D1=81=D0=BA,=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8=D1=87=D0=BD?= =?UTF-8?q?=D1=8B=D0=B5=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B8?= =?UTF-8?q?,=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=20=D0=BA=D0=B0=D0=B6=D0=B4?= =?UTF-8?q?=D0=BE=D0=B9=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8,=20?= =?UTF-8?q?=D1=80=D0=B5=D1=81=D1=82=D0=B0=D1=80=D1=82=20=D1=81=D1=82=D0=B5?= =?UTF-8?q?=D0=BD=D0=B4=D0=B0,=20=D0=B3=D0=BE=D0=BD=D0=BA=D0=B8=20=D0=B8?= =?UTF-8?q?=20=D0=BE=D0=B1=D1=89=D0=B0=D1=8F=20=D0=B8=D0=B4=D0=B5=D0=BC?= =?UTF-8?q?=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D1=8C,=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D1=8B=20=D0=B8=20rolli?= =?UTF-8?q?ng=20update=20=D0=BF=D0=BE=D0=B4=20=D0=BD=D0=B0=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B7=D0=BA=D0=BE=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evidence/bootstrap.txt | 241 ++++++++++++++++++ evidence/concurrency.json | 26 ++ evidence/concurrency.txt | 36 +++ evidence/failover.txt | 127 +++++++++ evidence/failover/kill-1/faults.json | 9 + evidence/failover/kill-1/instances-after.txt | 3 + evidence/failover/kill-1/instances-before.txt | 3 + evidence/failover/kill-1/meta.json | 16 ++ evidence/failover/kill-1/seconds.csv | 61 +++++ evidence/failover/kill-1/timeline.json | 44 ++++ evidence/failover/kill-2/faults.json | 9 + evidence/failover/kill-2/instances-after.txt | 3 + evidence/failover/kill-2/instances-before.txt | 3 + evidence/failover/kill-2/meta.json | 16 ++ evidence/failover/kill-2/seconds.csv | 61 +++++ evidence/failover/kill-2/timeline.json | 44 ++++ evidence/failover/stop-1/faults.json | 9 + evidence/failover/stop-1/instances-after.txt | 3 + evidence/failover/stop-1/instances-before.txt | 3 + evidence/failover/stop-1/meta.json | 16 ++ evidence/failover/stop-1/seconds.csv | 61 +++++ evidence/failover/stop-1/timeline.json | 44 ++++ evidence/failover/stop-2/faults.json | 9 + evidence/failover/stop-2/instances-after.txt | 3 + evidence/failover/stop-2/instances-before.txt | 3 + evidence/failover/stop-2/meta.json | 16 ++ evidence/failover/stop-2/seconds.csv | 61 +++++ evidence/failover/stop-2/timeline.json | 44 ++++ evidence/persistence.txt | 23 ++ evidence/rollout/instances-after.txt | 3 + evidence/rollout/instances-before.txt | 3 + evidence/rollout/meta.json | 13 + evidence/rollout/seconds.csv | 61 +++++ evidence/rollout/timeline.json | 45 ++++ evidence/rollout/update.json | 7 + evidence/smoke.txt | 13 + evidence/tests.txt | 179 +++++++++++++ 37 files changed, 1321 insertions(+) create mode 100644 evidence/bootstrap.txt create mode 100644 evidence/concurrency.json create mode 100644 evidence/concurrency.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failover/kill-1/faults.json create mode 100644 evidence/failover/kill-1/instances-after.txt create mode 100644 evidence/failover/kill-1/instances-before.txt create mode 100644 evidence/failover/kill-1/meta.json create mode 100644 evidence/failover/kill-1/seconds.csv create mode 100644 evidence/failover/kill-1/timeline.json create mode 100644 evidence/failover/kill-2/faults.json create mode 100644 evidence/failover/kill-2/instances-after.txt create mode 100644 evidence/failover/kill-2/instances-before.txt create mode 100644 evidence/failover/kill-2/meta.json create mode 100644 evidence/failover/kill-2/seconds.csv create mode 100644 evidence/failover/kill-2/timeline.json create mode 100644 evidence/failover/stop-1/faults.json create mode 100644 evidence/failover/stop-1/instances-after.txt create mode 100644 evidence/failover/stop-1/instances-before.txt create mode 100644 evidence/failover/stop-1/meta.json create mode 100644 evidence/failover/stop-1/seconds.csv create mode 100644 evidence/failover/stop-1/timeline.json create mode 100644 evidence/failover/stop-2/faults.json create mode 100644 evidence/failover/stop-2/instances-after.txt create mode 100644 evidence/failover/stop-2/instances-before.txt create mode 100644 evidence/failover/stop-2/meta.json create mode 100644 evidence/failover/stop-2/seconds.csv create mode 100644 evidence/failover/stop-2/timeline.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/rollout/instances-after.txt create mode 100644 evidence/rollout/instances-before.txt create mode 100644 evidence/rollout/meta.json create mode 100644 evidence/rollout/seconds.csv create mode 100644 evidence/rollout/timeline.json create mode 100644 evidence/rollout/update.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/tests.txt diff --git a/evidence/bootstrap.txt b/evidence/bootstrap.txt new file mode 100644 index 00000000..b40df4ae --- /dev/null +++ b/evidence/bootstrap.txt @@ -0,0 +1,241 @@ +# Bootstrap from empty volumes: config, start, migrations, schema, Valkey, balancer, replicas +# date (UTC): 2026-11-04T09:59:38Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +$ docker compose down -v --remove-orphans + Container lab3-api-1 Stopping + Container lab3-api-2 Stopping + Container lab3-lb-1 Stopping + Container lab3-lb-1 Stopped + Container lab3-lb-1 Removing + Container lab3-lb-1 Removed + Container lab3-api-1 Stopped + Container lab3-api-1 Removing + Container lab3-api-1 Removed + Container lab3-api-2 Stopped + Container lab3-api-2 Removing + Container lab3-api-2 Removed + Container lab3-valkey-1 Stopping + Container lab3-migrate-1 Stopping + Container lab3-migrate-1 Stopped + Container lab3-migrate-1 Removing + Container lab3-migrate-1 Removed + Container lab3-postgres-1 Stopping + Container lab3-valkey-1 Stopped + Container lab3-valkey-1 Removing + Container lab3-valkey-1 Removed + Container lab3-postgres-1 Stopped + Container lab3-postgres-1 Removing + Container lab3-postgres-1 Removed + Network lab3_data Removing + Network lab3_edge Removing + Volume lab3_valkeydata Removing + Volume lab3_pgdata Removing + Volume lab3_valkeydata Removed + Volume lab3_pgdata Removed + Network lab3_data Removed + Network lab3_edge Removed + +$ docker compose config --quiet + +(config --quiet: exit 0) + +$ docker compose up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 500B 0.0s done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.73kB 0.0s done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 +#3 DONE 0.0s + +#4 [internal] load metadata for docker.io/library/golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 +#4 DONE 0.0s + +#5 [internal] load .dockerignore +#5 transferring context: 224B 0.0s done +#5 DONE 0.0s + +#6 [build 1/8] FROM docker.io/library/golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 +#6 DONE 0.0s + +#7 [stage-3 1/3] FROM docker.io/library/alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 4.25kB 0.0s done +#8 DONE 0.0s + +#9 [build 4/8] RUN go mod download && go mod verify +#9 CACHED + +#10 [stage-3 2/3] RUN adduser -D -H -u 10001 app +#10 CACHED + +#11 [build 2/8] WORKDIR /src +#11 CACHED + +#12 [build 8/8] RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate +#12 CACHED + +#13 [build 5/8] COPY cmd ./cmd +#13 CACHED + +#14 [build 6/8] COPY internal ./internal +#14 CACHED + +#15 [build 7/8] COPY migrations ./migrations +#15 CACHED + +#16 [build 3/8] COPY go.mod go.sum ./ +#16 CACHED + +#17 [stage-3 3/3] COPY --from=build /out/server /out/migrate /usr/local/bin/ +#17 CACHED + +#18 exporting to image +#18 exporting layers done +#18 writing image sha256:74f0bfa454274aab5fe0abbf928282103666140ef3588a5910a771e83d53b2d5 done +#18 naming to docker.io/library/mtusi-messenger:lab3 done +#18 DONE 0.0s + +#19 resolving provenance for metadata file +#19 DONE 0.0s + mtusi-messenger:lab3 Built + Network lab3_data Creating + Network lab3_data Created + Network lab3_edge Creating + Network lab3_edge Created + Volume lab3_pgdata Creating + Volume lab3_pgdata Created + Volume lab3_valkeydata Creating + Volume lab3_valkeydata Created + Container lab3-postgres-1 Creating + Container lab3-valkey-1 Creating + Container lab3-lb-1 Creating + Container lab3-lb-1 Created + Container lab3-postgres-1 Created + Container lab3-migrate-1 Creating + Container lab3-valkey-1 Created + Container lab3-migrate-1 Created + Container lab3-api-2 Creating + Container lab3-api-1 Creating + Container lab3-api-2 Created + Container lab3-api-1 Created + Container lab3-lb-1 Starting + Container lab3-valkey-1 Starting + Container lab3-postgres-1 Starting + Container lab3-postgres-1 Started + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Started + Container lab3-lb-1 Started + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Starting + Container lab3-migrate-1 Started + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Waiting + Container lab3-migrate-1 Waiting + Container lab3-valkey-1 Healthy + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Exited + Container lab3-api-1 Starting + Container lab3-api-1 Started + Container lab3-api-2 Starting + Container lab3-api-2 Started + Container lab3-api-1 Waiting + Container lab3-api-2 Waiting + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Waiting + Container lab3-lb-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-valkey-1 Healthy + Container lab3-migrate-1 Exited + Container lab3-lb-1 Healthy + Container lab3-api-1 Healthy + Container lab3-api-2 Healthy + +$ docker compose ps --format table {{.Name}}\t{{.Status}}\t{{.Ports}} +NAME STATUS PORTS +lab3-api-1 Up 2 seconds (healthy) 8080/tcp +lab3-api-2 Up 2 seconds (healthy) 8080/tcp +lab3-lb-1 Up 9 seconds (healthy) 127.0.0.1:8404->8404/tcp, 127.0.0.1:28080->8080/tcp +lab3-postgres-1 Up 9 seconds (healthy) +lab3-valkey-1 Up 9 seconds (healthy) + +# Only lb publishes ports; api replicas have none. +$ docker compose logs --no-color migrate +migrate-1 | {"time":"2026-11-04T09:59:54.091643913Z","level":"INFO","msg":"migrations applied","applied_migrations":[1,2]} + +$ docker compose exec -T postgres psql -U messenger -d messenger -X -c \dt -c \di + List of tables + Schema | Name | Type | Owner +--------+--------------------------+-------+----------- + public | chat_members | table | messenger + public | chats | table | messenger + public | goose_db_version | table | messenger + public | message_idempotency_keys | table | messenger + public | messages | table | messenger + public | users | table | messenger +(6 rows) + + List of indexes + Schema | Name | Type | Owner | Table +--------+-----------------------------------------+-------+-----------+-------------------------- + public | chat_members_pkey | index | messenger | chat_members + public | chat_members_user_chat_idx | index | messenger | chat_members + public | chats_pkey | index | messenger | chats + public | goose_db_version_pkey | index | messenger | goose_db_version + public | message_idempotency_keys_created_at_idx | index | messenger | message_idempotency_keys + public | message_idempotency_keys_pkey | index | messenger | message_idempotency_keys + public | messages_pkey | index | messenger | messages + public | users_pkey | index | messenger | users + public | users_username_key | index | messenger | users +(9 rows) + + +$ valkey-cli CONFIG GET appendonly appendfsync maxmemory-policy (password from the secret file) +appendonly +yes +appendfsync +always +maxmemory-policy +noeviction + +$ curl http://127.0.0.1:8404/;csv (balancer servers: name, state, maxconn limit, check status) +api1 UP slim=20 L7OK +api2 UP slim=20 L7OK +api3 MAINT (resolution) slim=20 +api4 MAINT (resolution) slim=20 +api5 MAINT (resolution) slim=20 +api6 MAINT (resolution) slim=20 +api7 MAINT (resolution) slim=20 +api8 MAINT (resolution) slim=20 +api9 MAINT (resolution) slim=20 +api10 MAINT (resolution) slim=20 +BACKEND UP slim=2000 + +# One session, ten reads through the balancer: X-Instance-Id of each answer +request 1: 200 X-Instance-Id c6f9836eb2492e55 +request 2: 200 X-Instance-Id 0d2f4075b61465bd +request 3: 200 X-Instance-Id c6f9836eb2492e55 +request 4: 200 X-Instance-Id 0d2f4075b61465bd +request 5: 200 X-Instance-Id c6f9836eb2492e55 +request 6: 200 X-Instance-Id 0d2f4075b61465bd +request 7: 200 X-Instance-Id c6f9836eb2492e55 +request 8: 200 X-Instance-Id 0d2f4075b61465bd +request 9: 200 X-Instance-Id c6f9836eb2492e55 +request 10: 200 X-Instance-Id 0d2f4075b61465bd + +$ sh scripts/instances.sh +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +2e2108cc993f lab3-api-1 running 0d2f4075b61465bd 74f0bfa45427 +c8e0ac4a0557 lab3-api-2 running c6f9836eb2492e55 74f0bfa45427 + +# A client X-Request-Id is replaced by the balancer's id, which the API keeps: +x-instance-id: 0d2f4075b61465bd +x-request-id: 94b2c715b6374efbb13facad950213d5 diff --git a/evidence/concurrency.json b/evidence/concurrency.json new file mode 100644 index 00000000..5f6d782b --- /dev/null +++ b/evidence/concurrency.json @@ -0,0 +1,26 @@ +{ + "base_url": "http://localhost:28080", + "started_at": "2026-11-04T10:01:16.0582191Z", + "clients": 10, + "per_client": 10, + "expected_messages": 100, + "created_201": 100, + "failed_posts": 0, + "listed_messages": 100, + "unique_ids": 100, + "created_but_not_listed": 0, + "history_ordered_by_id": true, + "pages_read": 1, + "elapsed_seconds": 0.5357041, + "idempotency": { + "rounds": 20, + "parallel": 8, + "same_message_rounds": 20, + "instances_seen": 2, + "require_two_replicas": true, + "different_body_409": true, + "history_messages": 20, + "passed": true + }, + "passed": true +} diff --git a/evidence/concurrency.txt b/evidence/concurrency.txt new file mode 100644 index 00000000..1863d96b --- /dev/null +++ b/evidence/concurrency.txt @@ -0,0 +1,36 @@ +# make concurrency: phase 1 races of sends, phase 2 one Idempotency-Key in parallel through the balancer +# date (UTC): 2026-11-04T10:01:14Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +$ make concurrency BASE_URL=http://localhost:28080 EVIDENCE_DIR=evidence +go run ./tools/concurrency --base-url "http://localhost:28080" --output "evidence/concurrency.json" +{ + "base_url": "http://localhost:28080", + "started_at": "2026-11-04T10:01:16.0582191Z", + "clients": 10, + "per_client": 10, + "expected_messages": 100, + "created_201": 100, + "failed_posts": 0, + "listed_messages": 100, + "unique_ids": 100, + "created_but_not_listed": 0, + "history_ordered_by_id": true, + "pages_read": 1, + "elapsed_seconds": 0.5357041, + "idempotency": { + "rounds": 20, + "parallel": 8, + "same_message_rounds": 20, + "instances_seen": 2, + "require_two_replicas": true, + "different_body_409": true, + "history_messages": 20, + "passed": true + }, + "passed": true +} +PASS: 10 clients x 10 messages produced exactly 100 unique ordered messages +PASS: 20 rounds x 8 parallel attempts with one key produced 20 messages across 2 instances + diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..206a84aa --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,127 @@ +# make test-failover with scripts/stop-one.cmd (Windows wrapper of stop-one.sh), each replica in turn +# date (UTC): 2026-11-04T10:00:04Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +## Replica 1 +$ sh scripts/instances.sh +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +2e2108cc993f lab3-api-1 running 0d2f4075b61465bd 74f0bfa45427 +c8e0ac4a0557 lab3-api-2 running c6f9836eb2492e55 74f0bfa45427 + +$ make test-failover PYTHON=python BASE_URL=http://localhost:28080 ACTION_SCRIPT=scripts/stop-one.cmd +python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:28080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.572} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# Which container the hook stopped: +$ sh scripts/instances.sh +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +2e2108cc993f lab3-api-1 running 0d2f4075b61465bd 74f0bfa45427 +c8e0ac4a0557 lab3-api-2 exited - 74f0bfa45427 + +$ docker compose start api + Container lab3-postgres-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Starting + Container lab3-migrate-1 Started + Container lab3-valkey-1 Waiting + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Healthy + Container lab3-migrate-1 Exited + Container lab3-postgres-1 Healthy + Container lab3-api-2 Starting + Container lab3-api-2 Started + +$ docker compose up -d --wait + Container lab3-postgres-1 Running + Container lab3-lb-1 Running + Container lab3-valkey-1 Running + Container lab3-api-1 Running + Container lab3-api-2 Running + Container lab3-postgres-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Starting + Container lab3-migrate-1 Started + Container lab3-valkey-1 Waiting + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-migrate-1 Exited + Container lab3-valkey-1 Healthy + Container lab3-postgres-1 Healthy + Container lab3-lb-1 Waiting + Container lab3-api-1 Waiting + Container lab3-api-2 Waiting + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Waiting + Container lab3-migrate-1 Exited + Container lab3-postgres-1 Healthy + Container lab3-valkey-1 Healthy + Container lab3-lb-1 Healthy + Container lab3-api-1 Healthy + Container lab3-api-2 Healthy + +## Replica 2 +$ sh scripts/instances.sh +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +2e2108cc993f lab3-api-1 running 0d2f4075b61465bd 74f0bfa45427 +c8e0ac4a0557 lab3-api-2 running 33ecf420baa52453 74f0bfa45427 + +$ make test-failover PYTHON=python BASE_URL=http://localhost:28080 ACTION_SCRIPT=scripts/stop-one.cmd +python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:28080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 5.188} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# Which container the hook stopped: +$ sh scripts/instances.sh +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +2e2108cc993f lab3-api-1 exited - 74f0bfa45427 +c8e0ac4a0557 lab3-api-2 running 33ecf420baa52453 74f0bfa45427 + +$ docker compose start api + Container lab3-postgres-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Starting + Container lab3-migrate-1 Started + Container lab3-valkey-1 Waiting + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-migrate-1 Exited + Container lab3-valkey-1 Healthy + Container lab3-postgres-1 Healthy + Container lab3-api-1 Starting + Container lab3-api-1 Started + +$ docker compose up -d --wait + Container lab3-postgres-1 Running + Container lab3-valkey-1 Running + Container lab3-lb-1 Running + Container lab3-api-1 Running + Container lab3-api-2 Running + Container lab3-postgres-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Starting + Container lab3-migrate-1 Started + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Waiting + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Exited + Container lab3-valkey-1 Healthy + Container lab3-migrate-1 Waiting + Container lab3-postgres-1 Waiting + Container lab3-valkey-1 Waiting + Container lab3-lb-1 Waiting + Container lab3-api-1 Waiting + Container lab3-api-2 Waiting + Container lab3-lb-1 Healthy + Container lab3-postgres-1 Healthy + Container lab3-migrate-1 Exited + Container lab3-api-1 Healthy + Container lab3-api-2 Healthy + Container lab3-valkey-1 Healthy + diff --git a/evidence/failover/kill-1/faults.json b/evidence/failover/kill-1/faults.json new file mode 100644 index 00000000..86f9e271 --- /dev/null +++ b/evidence/failover/kill-1/faults.json @@ -0,0 +1,9 @@ +{ + "kind": "kill", + "container": "5bcf254ae6ba", + "x_instance_id": "74fa1c12ba4b1c72", + "process_exited_vm_utc": "2026-11-04T10:12:30.516996427Z", + "restarted_vm_utc": "2026-11-04T10:12:52.839612581Z", + "fault_command_host_utc": "2026-11-04T10:12:30.322Z", + "restore_command_host_utc": "2026-11-04T10:12:51.353Z" +} diff --git a/evidence/failover/kill-1/instances-after.txt b/evidence/failover/kill-1/instances-after.txt new file mode 100644 index 00000000..add5a5b7 --- /dev/null +++ b/evidence/failover/kill-1/instances-after.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 6e898c3a3552ad08 74f0bfa45427 +c6c9938a148f lab3-api-4 running 02253d7573081a60 74f0bfa45427 diff --git a/evidence/failover/kill-1/instances-before.txt b/evidence/failover/kill-1/instances-before.txt new file mode 100644 index 00000000..0c90d056 --- /dev/null +++ b/evidence/failover/kill-1/instances-before.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 74fa1c12ba4b1c72 74f0bfa45427 +c6c9938a148f lab3-api-4 running 02253d7573081a60 74f0bfa45427 diff --git a/evidence/failover/kill-1/meta.json b/evidence/failover/kill-1/meta.json new file mode 100644 index 00000000..a6ad91e0 --- /dev/null +++ b/evidence/failover/kill-1/meta.json @@ -0,0 +1,16 @@ +{ + "date_utc": "2026-11-04T10:12:12Z", + "commit": "1626145", + "branch": "solution/bpi2301-bakulin", + "docker": "28.5.1", + "host_cpus": "8", + "host_memory_bytes": "12547280896", + "experiment": "failover", + "kind": "kill", + "replica": "1", + "rate": "50", + "duration": "60s", + "fault_at_s": "15", + "restore_at_s": "35", + "go": "go1.27.1" +} diff --git a/evidence/failover/kill-1/seconds.csv b/evidence/failover/kill-1/seconds.csv new file mode 100644 index 00000000..b642db31 --- /dev/null +++ b/evidence/failover/kill-1/seconds.csv @@ -0,0 +1,61 @@ +second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances +0,50,50,0,0,0,0,0,0,4.7,9.1,12.7,1.3,02253d7573081a60=25|74fa1c12ba4b1c72=25 +1,50,50,0,0,0,0,0,0,6.0,15.5,40.1,1.3,02253d7573081a60=25|74fa1c12ba4b1c72=25 +2,50,50,0,0,0,0,0,0,5.6,8.0,10.3,1.3,02253d7573081a60=25|74fa1c12ba4b1c72=25 +3,50,50,0,0,0,0,0,0,4.0,8.0,10.7,1.4,02253d7573081a60=26|74fa1c12ba4b1c72=24 +4,50,50,0,0,0,0,0,0,2.9,4.6,4.8,1.1,02253d7573081a60=25|74fa1c12ba4b1c72=25 +5,50,50,0,0,0,0,0,0,3.0,4.6,6.2,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +6,50,50,0,0,0,0,0,0,3.2,5.3,8.6,1.1,02253d7573081a60=25|74fa1c12ba4b1c72=25 +7,50,50,0,0,0,0,0,0,2.8,4.2,4.9,1.7,02253d7573081a60=25|74fa1c12ba4b1c72=25 +8,50,50,0,0,0,0,0,0,3.0,4.2,4.9,1.3,02253d7573081a60=24|74fa1c12ba4b1c72=26 +9,50,50,0,0,0,0,0,0,3.0,4.5,5.4,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +10,50,50,0,0,0,0,0,0,3.3,5.3,6.0,1.4,02253d7573081a60=25|74fa1c12ba4b1c72=25 +11,50,50,0,0,0,0,0,0,3.1,5.0,5.1,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +12,50,50,0,0,0,0,0,0,3.1,4.3,5.1,2.5,02253d7573081a60=25|74fa1c12ba4b1c72=25 +13,50,50,0,0,0,0,0,0,3.4,5.2,5.8,1.2,02253d7573081a60=26|74fa1c12ba4b1c72=24 +14,50,50,0,0,0,0,0,0,3.9,1005.3,1012.4,1.2,02253d7573081a60=39|74fa1c12ba4b1c72=11 +15,50,50,0,0,0,0,0,0,4.2,1004.2,1004.7,1.2,02253d7573081a60=50 +16,50,50,0,0,0,0,0,0,3.1,4.8,6.6,1.3,02253d7573081a60=50 +17,50,50,0,0,0,0,0,0,3.0,4.2,6.8,1.9,02253d7573081a60=50 +18,50,50,0,0,0,0,0,0,3.1,4.5,6.9,1.2,02253d7573081a60=50 +19,50,50,0,0,0,0,0,0,3.1,4.2,5.3,1.2,02253d7573081a60=50 +20,50,50,0,0,0,0,0,0,3.4,5.5,6.4,1.4,02253d7573081a60=50 +21,50,50,0,0,0,0,0,0,2.9,4.5,4.9,1.2,02253d7573081a60=50 +22,50,50,0,0,0,0,0,0,3.0,4.1,4.8,1.2,02253d7573081a60=50 +23,50,50,0,0,0,0,0,0,3.5,4.5,5.0,1.4,02253d7573081a60=50 +24,50,50,0,0,0,0,0,0,2.9,3.9,4.9,1.3,02253d7573081a60=50 +25,50,50,0,0,0,0,0,0,3.2,5.5,7.6,1.3,02253d7573081a60=50 +26,50,50,0,0,0,0,0,0,3.0,3.9,5.0,1.3,02253d7573081a60=50 +27,50,50,0,0,0,0,0,0,3.2,4.6,6.0,1.4,02253d7573081a60=50 +28,50,50,0,0,0,0,0,0,3.1,5.2,8.5,1.1,02253d7573081a60=50 +29,50,50,0,0,0,0,0,0,3.1,4.6,5.7,1.2,02253d7573081a60=50 +30,50,50,0,0,0,0,0,0,3.2,5.7,6.4,4.6,02253d7573081a60=50 +31,50,50,0,0,0,0,0,0,3.0,5.0,8.9,1.2,02253d7573081a60=50 +32,50,50,0,0,0,0,0,0,2.9,4.6,5.4,1.3,02253d7573081a60=50 +33,50,50,0,0,0,0,0,0,3.5,4.8,5.2,1.4,02253d7573081a60=50 +34,50,50,0,0,0,0,0,0,3.1,4.1,4.4,1.3,02253d7573081a60=50 +35,50,50,0,0,0,0,0,0,3.5,5.3,5.8,1.3,02253d7573081a60=50 +36,50,50,0,0,0,0,0,0,3.7,5.5,6.3,1.3,02253d7573081a60=50 +37,50,50,0,0,0,0,0,0,4.1,6.0,9.4,1.2,02253d7573081a60=31|6e898c3a3552ad08=19 +38,50,50,0,0,0,0,0,0,3.4,4.6,4.9,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +39,50,50,0,0,0,0,0,0,3.0,4.6,5.0,1.9,02253d7573081a60=25|6e898c3a3552ad08=25 +40,50,50,0,0,0,0,0,0,3.3,5.1,5.3,1.5,02253d7573081a60=25|6e898c3a3552ad08=25 +41,50,50,0,0,0,0,0,0,3.0,4.2,7.1,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +42,50,50,0,0,0,0,0,0,2.9,4.1,8.1,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +43,50,50,0,0,0,0,0,0,3.7,5.4,6.6,1.3,02253d7573081a60=25|6e898c3a3552ad08=25 +44,50,50,0,0,0,0,0,0,3.5,4.6,5.3,1.3,02253d7573081a60=25|6e898c3a3552ad08=25 +45,50,50,0,0,0,0,0,0,3.0,4.6,5.6,2.7,02253d7573081a60=25|6e898c3a3552ad08=25 +46,50,50,0,0,0,0,0,0,3.0,4.1,4.3,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +47,50,50,0,0,0,0,0,0,2.9,4.3,5.7,1.1,02253d7573081a60=25|6e898c3a3552ad08=25 +48,50,50,0,0,0,0,0,0,3.0,4.4,5.6,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +49,50,50,0,0,0,0,0,0,3.1,4.7,6.0,2.0,02253d7573081a60=25|6e898c3a3552ad08=25 +50,50,50,0,0,0,0,0,0,3.4,6.2,8.5,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +51,50,50,0,0,0,0,0,0,3.3,5.2,6.5,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +52,50,50,0,0,0,0,0,0,3.2,4.9,5.6,1.5,02253d7573081a60=25|6e898c3a3552ad08=25 +53,50,50,0,0,0,0,0,0,3.8,5.5,14.4,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +54,50,50,0,0,0,0,0,0,3.2,4.9,6.6,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +55,50,50,0,0,0,0,0,0,3.1,4.7,6.2,1.5,02253d7573081a60=25|6e898c3a3552ad08=25 +56,50,50,0,0,0,0,0,0,3.2,4.4,6.4,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +57,50,50,0,0,0,0,0,0,2.9,4.0,4.6,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +58,50,50,0,0,0,0,0,0,3.0,3.8,4.0,1.1,02253d7573081a60=25|6e898c3a3552ad08=25 +59,50,50,0,0,0,0,0,0,2.9,4.1,5.5,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 diff --git a/evidence/failover/kill-1/timeline.json b/evidence/failover/kill-1/timeline.json new file mode 100644 index 00000000..d8611ba8 --- /dev/null +++ b/evidence/failover/kill-1/timeline.json @@ -0,0 +1,44 @@ +{ + "started_at": "2026-11-04T10:12:16.087828449Z", + "config": { + "mode": "timeline", + "base_url": "http://lb:8080", + "users": 200, + "chats": 100, + "seed_messages_per_chat": 50, + "read_ratio": 0.8, + "rates": [ + 200, + 400 + ], + "step": "30s", + "warmup": "30s", + "rate": 50, + "duration": "60s", + "client_timeout": "2s", + "seed": 1, + "scrape": false, + "stop_on_fail": false, + "stats_url": "http://lb:8404/;csv", + "api_host": "api", + "debug_port": "9090" + }, + "messages_attempted": 3000, + "requests": 3000, + "confirmed": 3000, + "messages_retried": 0, + "unresolved": 0, + "rejected_4xx": 0, + "classes": { + "ok": 3000 + }, + "instances": { + "02253d7573081a60": 2071, + "6e898c3a3552ad08": 569, + "74fa1c12ba4b1c72": 360 + }, + "history_messages": 3000, + "duplicates": 0, + "missing_confirmed": 0, + "passed": true +} diff --git a/evidence/failover/kill-2/faults.json b/evidence/failover/kill-2/faults.json new file mode 100644 index 00000000..7be01b57 --- /dev/null +++ b/evidence/failover/kill-2/faults.json @@ -0,0 +1,9 @@ +{ + "kind": "kill", + "container": "c6c9938a148f", + "x_instance_id": "02253d7573081a60", + "process_exited_vm_utc": "2026-11-04T10:13:41.478334864Z", + "restarted_vm_utc": "2026-11-04T10:14:03.728936284Z", + "fault_command_host_utc": "2026-11-04T10:13:41.276Z", + "restore_command_host_utc": "2026-11-04T10:14:02.223Z" +} diff --git a/evidence/failover/kill-2/instances-after.txt b/evidence/failover/kill-2/instances-after.txt new file mode 100644 index 00000000..13dc7cfa --- /dev/null +++ b/evidence/failover/kill-2/instances-after.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 6e898c3a3552ad08 74f0bfa45427 +c6c9938a148f lab3-api-4 running 0b71c96e22052a34 74f0bfa45427 diff --git a/evidence/failover/kill-2/instances-before.txt b/evidence/failover/kill-2/instances-before.txt new file mode 100644 index 00000000..add5a5b7 --- /dev/null +++ b/evidence/failover/kill-2/instances-before.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 6e898c3a3552ad08 74f0bfa45427 +c6c9938a148f lab3-api-4 running 02253d7573081a60 74f0bfa45427 diff --git a/evidence/failover/kill-2/meta.json b/evidence/failover/kill-2/meta.json new file mode 100644 index 00000000..1c646d45 --- /dev/null +++ b/evidence/failover/kill-2/meta.json @@ -0,0 +1,16 @@ +{ + "date_utc": "2026-11-04T10:13:23Z", + "commit": "1626145", + "branch": "solution/bpi2301-bakulin", + "docker": "28.5.1", + "host_cpus": "8", + "host_memory_bytes": "12547280896", + "experiment": "failover", + "kind": "kill", + "replica": "2", + "rate": "50", + "duration": "60s", + "fault_at_s": "15", + "restore_at_s": "35", + "go": "go1.27.1" +} diff --git a/evidence/failover/kill-2/seconds.csv b/evidence/failover/kill-2/seconds.csv new file mode 100644 index 00000000..bbf88394 --- /dev/null +++ b/evidence/failover/kill-2/seconds.csv @@ -0,0 +1,61 @@ +second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances +0,50,50,0,0,0,0,0,0,4.0,5.2,8.7,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +1,50,50,0,0,0,0,0,0,3.6,5.1,5.9,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +2,50,50,0,0,0,0,0,0,3.6,4.6,5.1,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +3,50,50,0,0,0,0,0,0,3.4,5.1,6.9,1.2,02253d7573081a60=26|6e898c3a3552ad08=24 +4,50,50,0,0,0,0,0,0,3.1,5.0,6.0,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +5,50,50,0,0,0,0,0,0,4.9,8.8,12.5,1.3,02253d7573081a60=25|6e898c3a3552ad08=25 +6,50,50,0,0,0,0,0,0,6.0,8.3,18.8,1.6,02253d7573081a60=25|6e898c3a3552ad08=25 +7,50,50,0,0,0,0,0,0,3.7,7.9,8.8,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +8,50,50,0,0,0,0,0,0,3.2,4.5,5.5,1.2,02253d7573081a60=24|6e898c3a3552ad08=26 +9,50,50,0,0,0,0,0,0,3.2,5.4,7.2,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +10,50,50,0,0,0,0,0,0,3.3,5.9,7.7,1.3,02253d7573081a60=25|6e898c3a3552ad08=25 +11,50,50,0,0,0,0,0,0,3.4,5.1,8.1,1.4,02253d7573081a60=25|6e898c3a3552ad08=25 +12,50,50,0,0,0,0,0,0,3.8,5.2,6.4,1.2,02253d7573081a60=25|6e898c3a3552ad08=25 +13,50,50,0,0,0,0,0,0,3.1,4.0,6.2,1.8,02253d7573081a60=26|6e898c3a3552ad08=24 +14,50,50,0,0,0,0,0,0,4.2,1006.2,1015.5,1.4,02253d7573081a60=9|6e898c3a3552ad08=41 +15,50,50,0,0,0,0,0,0,4.2,1005.5,1007.1,1.3,6e898c3a3552ad08=50 +16,50,50,0,0,0,0,0,0,3.5,4.7,5.1,1.7,6e898c3a3552ad08=50 +17,50,50,0,0,0,0,0,0,3.4,5.4,5.5,1.2,6e898c3a3552ad08=50 +18,50,50,0,0,0,0,0,0,3.5,5.4,5.7,1.1,6e898c3a3552ad08=50 +19,50,50,0,0,0,0,0,0,3.5,5.3,5.9,2.0,6e898c3a3552ad08=50 +20,50,50,0,0,0,0,0,0,3.0,4.1,9.4,1.2,6e898c3a3552ad08=50 +21,50,50,0,0,0,0,0,0,3.1,5.7,6.8,1.2,6e898c3a3552ad08=50 +22,50,50,0,0,0,0,0,0,3.7,5.0,6.6,1.2,6e898c3a3552ad08=50 +23,50,50,0,0,0,0,0,0,3.0,4.3,6.8,1.2,6e898c3a3552ad08=50 +24,50,50,0,0,0,0,0,0,3.0,3.9,4.8,1.3,6e898c3a3552ad08=50 +25,50,50,0,0,0,0,0,0,2.9,3.8,4.4,1.7,6e898c3a3552ad08=50 +26,50,50,0,0,0,0,0,0,3.1,5.2,6.7,1.2,6e898c3a3552ad08=50 +27,50,50,0,0,0,0,0,0,3.2,4.4,6.5,1.2,6e898c3a3552ad08=50 +28,50,50,0,0,0,0,0,0,3.0,4.1,4.6,1.9,6e898c3a3552ad08=50 +29,50,50,0,0,0,0,0,0,3.4,5.9,6.4,1.3,6e898c3a3552ad08=50 +30,50,50,0,0,0,0,0,0,3.2,5.6,8.0,1.3,6e898c3a3552ad08=50 +31,50,50,0,0,0,0,0,0,3.2,5.1,18.0,1.2,6e898c3a3552ad08=50 +32,50,50,0,0,0,0,0,0,3.5,5.0,5.8,1.5,6e898c3a3552ad08=50 +33,50,50,0,0,0,0,0,0,2.8,4.4,10.5,1.2,6e898c3a3552ad08=50 +34,50,50,0,0,0,0,0,0,3.0,4.5,5.1,1.3,6e898c3a3552ad08=50 +35,50,50,0,0,0,0,0,0,3.1,5.0,5.2,1.3,6e898c3a3552ad08=50 +36,50,50,0,0,0,0,0,0,3.8,4.8,6.6,1.5,6e898c3a3552ad08=50 +37,50,50,0,0,0,0,0,0,3.7,5.0,7.8,1.3,0b71c96e22052a34=17|6e898c3a3552ad08=33 +38,50,50,0,0,0,0,0,0,3.6,4.6,5.0,1.2,0b71c96e22052a34=24|6e898c3a3552ad08=26 +39,50,50,0,0,0,0,0,0,3.4,9.0,10.2,1.4,0b71c96e22052a34=25|6e898c3a3552ad08=25 +40,50,50,0,0,0,0,0,0,3.0,4.5,5.2,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +41,50,50,0,0,0,0,0,0,3.0,4.9,5.2,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +42,50,50,0,0,0,0,0,0,3.6,5.0,6.2,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +43,50,50,0,0,0,0,0,0,3.1,4.2,4.7,1.2,0b71c96e22052a34=26|6e898c3a3552ad08=24 +44,50,50,0,0,0,0,0,0,3.1,5.0,6.1,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +45,50,50,0,0,0,0,0,0,2.9,4.2,4.9,1.4,0b71c96e22052a34=25|6e898c3a3552ad08=25 +46,50,50,0,0,0,0,0,0,2.9,4.3,5.4,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +47,50,50,0,0,0,0,0,0,3.1,4.5,9.3,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +48,50,50,0,0,0,0,0,0,3.2,4.5,4.7,1.3,0b71c96e22052a34=24|6e898c3a3552ad08=26 +49,50,50,0,0,0,0,0,0,3.4,4.9,5.5,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +50,50,50,0,0,0,0,0,0,3.2,5.9,8.8,1.8,0b71c96e22052a34=25|6e898c3a3552ad08=25 +51,50,50,0,0,0,0,0,0,2.9,4.5,8.2,1.4,0b71c96e22052a34=25|6e898c3a3552ad08=25 +52,50,50,0,0,0,0,0,0,3.3,4.3,5.5,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +53,50,50,0,0,0,0,0,0,3.1,4.5,5.1,1.1,0b71c96e22052a34=26|6e898c3a3552ad08=24 +54,50,50,0,0,0,0,0,0,3.6,4.4,5.2,1.7,0b71c96e22052a34=25|6e898c3a3552ad08=25 +55,50,50,0,0,0,0,0,0,3.5,5.1,7.9,2.0,0b71c96e22052a34=25|6e898c3a3552ad08=25 +56,50,50,0,0,0,0,0,0,3.1,4.9,5.5,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +57,50,50,0,0,0,0,0,0,2.9,4.0,5.2,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +58,50,50,0,0,0,0,0,0,3.0,3.8,5.4,1.1,0b71c96e22052a34=24|6e898c3a3552ad08=26 +59,50,50,0,0,0,0,0,0,3.0,5.1,5.9,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 diff --git a/evidence/failover/kill-2/timeline.json b/evidence/failover/kill-2/timeline.json new file mode 100644 index 00000000..69d3f457 --- /dev/null +++ b/evidence/failover/kill-2/timeline.json @@ -0,0 +1,44 @@ +{ + "started_at": "2026-11-04T10:13:27.105842841Z", + "config": { + "mode": "timeline", + "base_url": "http://lb:8080", + "users": 200, + "chats": 100, + "seed_messages_per_chat": 50, + "read_ratio": 0.8, + "rates": [ + 200, + 400 + ], + "step": "30s", + "warmup": "30s", + "rate": 50, + "duration": "60s", + "client_timeout": "2s", + "seed": 1, + "scrape": false, + "stop_on_fail": false, + "stats_url": "http://lb:8404/;csv", + "api_host": "api", + "debug_port": "9090" + }, + "messages_attempted": 3000, + "requests": 3000, + "confirmed": 3000, + "messages_retried": 0, + "unresolved": 0, + "rejected_4xx": 0, + "classes": { + "ok": 3000 + }, + "instances": { + "02253d7573081a60": 360, + "0b71c96e22052a34": 566, + "6e898c3a3552ad08": 2074 + }, + "history_messages": 3000, + "duplicates": 0, + "missing_confirmed": 0, + "passed": true +} diff --git a/evidence/failover/stop-1/faults.json b/evidence/failover/stop-1/faults.json new file mode 100644 index 00000000..b433e620 --- /dev/null +++ b/evidence/failover/stop-1/faults.json @@ -0,0 +1,9 @@ +{ + "kind": "stop", + "container": "5bcf254ae6ba", + "x_instance_id": "0535abc31194d205", + "process_exited_vm_utc": "2026-11-04T10:10:12.177073796Z", + "restarted_vm_utc": "2026-11-04T10:10:34.551307762Z", + "fault_command_host_utc": "2026-11-04T10:10:09.994Z", + "restore_command_host_utc": "2026-11-04T10:10:33.030Z" +} diff --git a/evidence/failover/stop-1/instances-after.txt b/evidence/failover/stop-1/instances-after.txt new file mode 100644 index 00000000..2aa0cdbb --- /dev/null +++ b/evidence/failover/stop-1/instances-after.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 74fa1c12ba4b1c72 74f0bfa45427 +c6c9938a148f lab3-api-4 running 2412d98d29ca4f3f 74f0bfa45427 diff --git a/evidence/failover/stop-1/instances-before.txt b/evidence/failover/stop-1/instances-before.txt new file mode 100644 index 00000000..eff42f91 --- /dev/null +++ b/evidence/failover/stop-1/instances-before.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 0535abc31194d205 74f0bfa45427 +c6c9938a148f lab3-api-4 running 2412d98d29ca4f3f 74f0bfa45427 diff --git a/evidence/failover/stop-1/meta.json b/evidence/failover/stop-1/meta.json new file mode 100644 index 00000000..45c6ab21 --- /dev/null +++ b/evidence/failover/stop-1/meta.json @@ -0,0 +1,16 @@ +{ + "date_utc": "2026-11-04T10:09:51Z", + "commit": "1626145", + "branch": "solution/bpi2301-bakulin", + "docker": "28.5.1", + "host_cpus": "8", + "host_memory_bytes": "12547280896", + "experiment": "failover", + "kind": "stop", + "replica": "1", + "rate": "50", + "duration": "60s", + "fault_at_s": "15", + "restore_at_s": "35", + "go": "go1.27.1" +} diff --git a/evidence/failover/stop-1/seconds.csv b/evidence/failover/stop-1/seconds.csv new file mode 100644 index 00000000..e3f88b0d --- /dev/null +++ b/evidence/failover/stop-1/seconds.csv @@ -0,0 +1,61 @@ +second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances +0,50,50,0,0,0,0,0,0,3.9,6.1,6.2,1.3,0535abc31194d205=25|2412d98d29ca4f3f=25 +1,50,50,0,0,0,0,0,0,4.0,5.5,8.2,1.5,0535abc31194d205=25|2412d98d29ca4f3f=25 +2,50,50,0,0,0,0,0,0,4.3,7.6,13.3,1.2,0535abc31194d205=24|2412d98d29ca4f3f=26 +3,50,50,0,0,0,0,0,0,3.9,5.3,5.5,1.3,0535abc31194d205=25|2412d98d29ca4f3f=25 +4,50,50,0,0,0,0,0,0,3.2,5.9,6.3,1.1,0535abc31194d205=25|2412d98d29ca4f3f=25 +5,50,50,0,0,0,0,0,0,3.4,5.8,8.0,1.2,0535abc31194d205=25|2412d98d29ca4f3f=25 +6,50,50,0,0,0,0,0,0,4.4,9.7,26.1,1.3,0535abc31194d205=25|2412d98d29ca4f3f=25 +7,50,50,0,0,0,0,0,0,4.6,7.9,9.1,1.3,0535abc31194d205=25|2412d98d29ca4f3f=25 +8,50,50,0,0,0,0,0,0,5.5,8.6,9.4,1.6,0535abc31194d205=25|2412d98d29ca4f3f=25 +9,50,50,0,0,0,0,0,0,3.1,4.2,4.7,1.6,0535abc31194d205=25|2412d98d29ca4f3f=25 +10,50,50,0,0,0,0,0,0,3.4,5.9,6.2,1.2,0535abc31194d205=25|2412d98d29ca4f3f=25 +11,50,50,0,0,0,0,0,0,3.5,7.2,13.6,2.7,0535abc31194d205=25|2412d98d29ca4f3f=25 +12,50,50,0,0,0,0,0,0,3.4,5.3,7.3,1.3,0535abc31194d205=25|2412d98d29ca4f3f=25 +13,50,50,0,0,0,0,0,0,3.5,5.7,8.0,1.9,0535abc31194d205=25|2412d98d29ca4f3f=25 +14,50,50,0,0,0,0,0,0,3.1,4.5,15.9,1.1,0535abc31194d205=25|2412d98d29ca4f3f=25 +15,50,50,0,0,0,0,0,0,3.2,5.0,5.8,1.6,0535abc31194d205=22|2412d98d29ca4f3f=28 +16,50,50,0,0,0,0,0,0,3.8,5.7,6.3,1.5,2412d98d29ca4f3f=50 +17,50,50,0,0,0,0,0,0,3.9,4.9,5.0,1.2,2412d98d29ca4f3f=50 +18,50,50,0,0,0,0,0,0,3.3,4.3,6.8,1.1,2412d98d29ca4f3f=50 +19,50,50,0,0,0,0,0,0,3.1,4.9,14.4,1.2,2412d98d29ca4f3f=50 +20,50,50,0,0,0,0,0,0,3.5,5.0,6.3,1.3,2412d98d29ca4f3f=50 +21,50,50,0,0,0,0,0,0,3.5,4.9,6.3,1.2,2412d98d29ca4f3f=50 +22,50,50,0,0,0,0,0,0,2.9,4.9,13.3,1.3,2412d98d29ca4f3f=50 +23,50,50,0,0,0,0,0,0,3.6,5.0,7.0,1.6,2412d98d29ca4f3f=50 +24,50,50,0,0,0,0,0,0,3.1,5.7,9.9,1.3,2412d98d29ca4f3f=50 +25,50,50,0,0,0,0,0,0,3.2,4.6,4.7,1.8,2412d98d29ca4f3f=50 +26,50,50,0,0,0,0,0,0,2.9,4.4,6.8,1.2,2412d98d29ca4f3f=50 +27,50,50,0,0,0,0,0,0,3.1,4.7,5.2,1.9,2412d98d29ca4f3f=50 +28,50,50,0,0,0,0,0,0,3.3,4.9,6.6,2.8,2412d98d29ca4f3f=50 +29,50,50,0,0,0,0,0,0,3.1,3.9,5.3,1.3,2412d98d29ca4f3f=50 +30,50,50,0,0,0,0,0,0,3.1,5.9,6.7,1.5,2412d98d29ca4f3f=50 +31,50,50,0,0,0,0,0,0,3.2,5.0,5.6,1.2,2412d98d29ca4f3f=50 +32,50,50,0,0,0,0,0,0,3.2,5.2,10.0,1.2,2412d98d29ca4f3f=50 +33,50,50,0,0,0,0,0,0,3.5,4.4,5.9,1.4,2412d98d29ca4f3f=50 +34,50,50,0,0,0,0,0,0,3.4,5.6,6.7,1.5,2412d98d29ca4f3f=50 +35,50,50,0,0,0,0,0,0,3.2,4.5,5.9,1.4,2412d98d29ca4f3f=50 +36,50,50,0,0,0,0,0,0,3.3,5.4,9.9,1.2,2412d98d29ca4f3f=50 +37,50,50,0,0,0,0,0,0,3.3,4.4,5.1,1.8,2412d98d29ca4f3f=50 +38,50,50,0,0,0,0,0,0,3.7,5.5,6.0,3.4,2412d98d29ca4f3f=50 +39,50,50,0,0,0,0,0,0,4.0,5.1,9.0,1.5,2412d98d29ca4f3f=35|74fa1c12ba4b1c72=15 +40,50,50,0,0,0,0,0,0,3.7,5.6,6.6,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +41,50,50,0,0,0,0,0,0,3.2,4.6,5.0,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +42,50,50,0,0,0,0,0,0,3.4,4.9,7.2,1.4,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +43,50,50,0,0,0,0,0,0,3.8,5.4,7.2,1.2,2412d98d29ca4f3f=24|74fa1c12ba4b1c72=26 +44,50,50,0,0,0,0,0,0,3.3,4.4,5.1,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +45,50,50,0,0,0,0,0,0,3.2,4.4,5.1,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +46,50,50,0,0,0,0,0,0,3.4,4.7,5.1,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +47,50,50,0,0,0,0,0,0,3.3,5.4,6.6,1.1,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +48,50,50,0,0,0,0,0,0,3.4,4.9,6.2,1.2,2412d98d29ca4f3f=26|74fa1c12ba4b1c72=24 +49,50,50,0,0,0,0,0,0,3.3,5.6,6.6,4.7,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +50,50,50,0,0,0,0,0,0,3.4,5.6,6.4,1.6,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +51,50,50,0,0,0,0,0,0,3.5,5.0,6.3,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +52,50,50,0,0,0,0,0,0,3.4,5.5,6.4,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +53,50,50,0,0,0,0,0,0,3.6,5.7,6.6,1.5,2412d98d29ca4f3f=24|74fa1c12ba4b1c72=26 +54,50,50,0,0,0,0,0,0,3.5,5.5,10.1,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +55,50,50,0,0,0,0,0,0,3.3,5.2,6.3,1.5,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +56,50,50,0,0,0,0,0,0,3.3,5.5,10.4,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +57,50,50,0,0,0,0,0,0,2.9,5.8,6.4,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +58,50,50,0,0,0,0,0,0,3.1,4.4,12.1,1.2,2412d98d29ca4f3f=26|74fa1c12ba4b1c72=24 +59,50,50,0,0,0,0,0,0,3.3,5.0,8.4,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 diff --git a/evidence/failover/stop-1/timeline.json b/evidence/failover/stop-1/timeline.json new file mode 100644 index 00000000..2127cda1 --- /dev/null +++ b/evidence/failover/stop-1/timeline.json @@ -0,0 +1,44 @@ +{ + "started_at": "2026-11-04T10:09:55.751121336Z", + "config": { + "mode": "timeline", + "base_url": "http://lb:8080", + "users": 200, + "chats": 100, + "seed_messages_per_chat": 50, + "read_ratio": 0.8, + "rates": [ + 200, + 400 + ], + "step": "30s", + "warmup": "30s", + "rate": 50, + "duration": "60s", + "client_timeout": "2s", + "seed": 1, + "scrape": false, + "stop_on_fail": false, + "stats_url": "http://lb:8404/;csv", + "api_host": "api", + "debug_port": "9090" + }, + "messages_attempted": 3000, + "requests": 3000, + "confirmed": 3000, + "messages_retried": 0, + "unresolved": 0, + "rejected_4xx": 0, + "classes": { + "ok": 3000 + }, + "instances": { + "0535abc31194d205": 396, + "2412d98d29ca4f3f": 2089, + "74fa1c12ba4b1c72": 515 + }, + "history_messages": 3000, + "duplicates": 0, + "missing_confirmed": 0, + "passed": true +} diff --git a/evidence/failover/stop-2/faults.json b/evidence/failover/stop-2/faults.json new file mode 100644 index 00000000..b62df5a3 --- /dev/null +++ b/evidence/failover/stop-2/faults.json @@ -0,0 +1,9 @@ +{ + "kind": "stop", + "container": "c6c9938a148f", + "x_instance_id": "2412d98d29ca4f3f", + "process_exited_vm_utc": "2026-11-04T10:11:22.892203716Z", + "restarted_vm_utc": "2026-11-04T10:11:45.183885307Z", + "fault_command_host_utc": "2026-11-04T10:11:20.689Z", + "restore_command_host_utc": "2026-11-04T10:11:43.678Z" +} diff --git a/evidence/failover/stop-2/instances-after.txt b/evidence/failover/stop-2/instances-after.txt new file mode 100644 index 00000000..0c90d056 --- /dev/null +++ b/evidence/failover/stop-2/instances-after.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 74fa1c12ba4b1c72 74f0bfa45427 +c6c9938a148f lab3-api-4 running 02253d7573081a60 74f0bfa45427 diff --git a/evidence/failover/stop-2/instances-before.txt b/evidence/failover/stop-2/instances-before.txt new file mode 100644 index 00000000..2aa0cdbb --- /dev/null +++ b/evidence/failover/stop-2/instances-before.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 74fa1c12ba4b1c72 74f0bfa45427 +c6c9938a148f lab3-api-4 running 2412d98d29ca4f3f 74f0bfa45427 diff --git a/evidence/failover/stop-2/meta.json b/evidence/failover/stop-2/meta.json new file mode 100644 index 00000000..865c0470 --- /dev/null +++ b/evidence/failover/stop-2/meta.json @@ -0,0 +1,16 @@ +{ + "date_utc": "2026-11-04T10:11:02Z", + "commit": "1626145", + "branch": "solution/bpi2301-bakulin", + "docker": "28.5.1", + "host_cpus": "8", + "host_memory_bytes": "12547280896", + "experiment": "failover", + "kind": "stop", + "replica": "2", + "rate": "50", + "duration": "60s", + "fault_at_s": "15", + "restore_at_s": "35", + "go": "go1.27.1" +} diff --git a/evidence/failover/stop-2/seconds.csv b/evidence/failover/stop-2/seconds.csv new file mode 100644 index 00000000..8b3ded20 --- /dev/null +++ b/evidence/failover/stop-2/seconds.csv @@ -0,0 +1,61 @@ +second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances +0,50,50,0,0,0,0,0,0,4.3,5.9,7.9,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +1,50,50,0,0,0,0,0,0,3.5,5.2,6.0,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +2,50,50,0,0,0,0,0,0,3.2,4.1,4.9,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +3,50,50,0,0,0,0,0,0,3.4,4.6,5.8,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +4,50,50,0,0,0,0,0,0,3.2,4.1,5.1,1.6,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +5,50,50,0,0,0,0,0,0,3.1,4.6,5.3,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +6,50,50,0,0,0,0,0,0,3.0,4.5,5.5,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +7,50,50,0,0,0,0,0,0,3.2,5.7,7.2,1.1,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +8,50,50,0,0,0,0,0,0,3.9,5.0,5.4,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +9,50,50,0,0,0,0,0,0,3.4,5.1,10.5,1.2,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +10,50,50,0,0,0,0,0,0,3.5,5.9,6.6,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +11,50,50,0,0,0,0,0,0,5.3,48.2,71.9,2.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +12,50,50,0,0,0,0,0,0,6.6,8.7,11.1,1.3,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +13,50,50,0,0,0,0,0,0,3.7,5.2,5.4,1.4,2412d98d29ca4f3f=26|74fa1c12ba4b1c72=24 +14,50,50,0,0,0,0,0,0,3.3,4.6,5.7,1.5,2412d98d29ca4f3f=25|74fa1c12ba4b1c72=25 +15,50,50,0,0,0,0,0,0,3.3,5.0,5.8,1.2,2412d98d29ca4f3f=11|74fa1c12ba4b1c72=39 +16,50,50,0,0,0,0,0,0,3.1,4.9,11.1,1.6,74fa1c12ba4b1c72=50 +17,50,50,0,0,0,0,0,0,3.8,5.0,6.8,1.3,74fa1c12ba4b1c72=50 +18,50,50,0,0,0,0,0,0,2.9,4.6,6.3,2.1,74fa1c12ba4b1c72=50 +19,50,50,0,0,0,0,0,0,3.3,5.0,6.3,1.8,74fa1c12ba4b1c72=50 +20,50,50,0,0,0,0,0,0,3.4,5.7,11.2,1.1,74fa1c12ba4b1c72=50 +21,50,50,0,0,0,0,0,0,3.0,5.6,6.0,1.2,74fa1c12ba4b1c72=50 +22,50,50,0,0,0,0,0,0,3.4,4.9,7.4,1.5,74fa1c12ba4b1c72=50 +23,50,50,0,0,0,0,0,0,3.3,4.7,5.4,2.0,74fa1c12ba4b1c72=50 +24,50,50,0,0,0,0,0,0,3.0,4.3,5.1,1.2,74fa1c12ba4b1c72=50 +25,50,50,0,0,0,0,0,0,2.9,4.3,6.3,1.8,74fa1c12ba4b1c72=50 +26,50,50,0,0,0,0,0,0,3.0,4.9,6.6,1.3,74fa1c12ba4b1c72=50 +27,50,50,0,0,0,0,0,0,3.1,4.9,7.4,2.2,74fa1c12ba4b1c72=50 +28,50,50,0,0,0,0,0,0,3.5,5.9,16.0,1.3,74fa1c12ba4b1c72=50 +29,50,50,0,0,0,0,0,0,3.2,4.8,5.2,1.4,74fa1c12ba4b1c72=50 +30,50,50,0,0,0,0,0,0,3.6,5.4,6.0,1.3,74fa1c12ba4b1c72=50 +31,50,50,0,0,0,0,0,0,3.2,5.2,13.9,1.2,74fa1c12ba4b1c72=50 +32,50,50,0,0,0,0,0,0,3.5,4.3,4.5,1.8,74fa1c12ba4b1c72=50 +33,50,50,0,0,0,0,0,0,3.2,4.7,6.1,1.3,74fa1c12ba4b1c72=50 +34,50,50,0,0,0,0,0,0,2.9,5.2,6.0,1.1,74fa1c12ba4b1c72=50 +35,50,50,0,0,0,0,0,0,3.1,5.3,6.4,1.2,74fa1c12ba4b1c72=50 +36,50,50,0,0,0,0,0,0,2.9,4.0,4.6,1.1,74fa1c12ba4b1c72=50 +37,50,50,0,0,0,0,0,0,3.5,4.4,4.7,1.3,74fa1c12ba4b1c72=50 +38,50,50,0,0,0,0,0,0,3.7,5.9,5.9,1.2,74fa1c12ba4b1c72=50 +39,50,50,0,0,0,0,0,0,3.9,5.1,8.2,1.2,02253d7573081a60=5|74fa1c12ba4b1c72=45 +40,50,50,0,0,0,0,0,0,4.1,5.7,7.0,3.4,02253d7573081a60=25|74fa1c12ba4b1c72=25 +41,50,50,0,0,0,0,0,0,3.0,4.1,5.0,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +42,50,50,0,0,0,0,0,0,3.2,4.7,5.8,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +43,50,50,0,0,0,0,0,0,3.4,4.9,5.6,1.2,02253d7573081a60=24|74fa1c12ba4b1c72=26 +44,50,50,0,0,0,0,0,0,3.1,5.3,7.4,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +45,50,50,0,0,0,0,0,0,3.1,4.2,5.3,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +46,50,50,0,0,0,0,0,0,3.0,5.1,5.4,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +47,50,50,0,0,0,0,0,0,2.9,4.8,9.4,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +48,50,50,0,0,0,0,0,0,3.1,4.1,7.9,1.8,02253d7573081a60=26|74fa1c12ba4b1c72=24 +49,50,50,0,0,0,0,0,0,3.2,4.5,4.6,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +50,50,50,0,0,0,0,0,0,3.7,5.7,6.2,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +51,50,50,0,0,0,0,0,0,3.3,5.1,17.3,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +52,50,50,0,0,0,0,0,0,3.3,5.7,14.6,1.4,02253d7573081a60=25|74fa1c12ba4b1c72=25 +53,50,50,0,0,0,0,0,0,3.2,4.3,5.1,1.4,02253d7573081a60=24|74fa1c12ba4b1c72=26 +54,50,50,0,0,0,0,0,0,2.9,3.8,5.9,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 +55,50,50,0,0,0,0,0,0,3.2,4.8,9.4,2.3,02253d7573081a60=25|74fa1c12ba4b1c72=25 +56,50,50,0,0,0,0,0,0,3.0,4.9,4.9,1.3,02253d7573081a60=25|74fa1c12ba4b1c72=25 +57,50,50,0,0,0,0,0,0,3.3,4.4,4.6,1.9,02253d7573081a60=25|74fa1c12ba4b1c72=25 +58,50,50,0,0,0,0,0,0,3.1,4.2,5.5,1.2,02253d7573081a60=26|74fa1c12ba4b1c72=24 +59,50,50,0,0,0,0,0,0,3.1,5.5,5.9,1.2,02253d7573081a60=25|74fa1c12ba4b1c72=25 diff --git a/evidence/failover/stop-2/timeline.json b/evidence/failover/stop-2/timeline.json new file mode 100644 index 00000000..03393450 --- /dev/null +++ b/evidence/failover/stop-2/timeline.json @@ -0,0 +1,44 @@ +{ + "started_at": "2026-11-04T10:11:06.426884535Z", + "config": { + "mode": "timeline", + "base_url": "http://lb:8080", + "users": 200, + "chats": 100, + "seed_messages_per_chat": 50, + "read_ratio": 0.8, + "rates": [ + 200, + 400 + ], + "step": "30s", + "warmup": "30s", + "rate": 50, + "duration": "60s", + "client_timeout": "2s", + "seed": 1, + "scrape": false, + "stop_on_fail": false, + "stats_url": "http://lb:8404/;csv", + "api_host": "api", + "debug_port": "9090" + }, + "messages_attempted": 3000, + "requests": 3000, + "confirmed": 3000, + "messages_retried": 0, + "unresolved": 0, + "rejected_4xx": 0, + "classes": { + "ok": 3000 + }, + "instances": { + "02253d7573081a60": 505, + "2412d98d29ca4f3f": 387, + "74fa1c12ba4b1c72": 2108 + }, + "history_messages": 3000, + "duplicates": 0, + "missing_confirmed": 0, + "passed": true +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..82dac8fd --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,23 @@ +# make test-persistence with scripts/restart.cmd: the whole stand restarts, volumes kept +# date (UTC): 2026-11-04T10:00:56Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +# StartedAt before: +/lab3-api-1 2026-11-04T10:00:53.341519776Z +/lab3-api-2 2026-11-04T10:00:28.728724752Z +/lab3-lb-1 2026-11-04T09:59:47.894746565Z +/lab3-postgres-1 2026-11-04T09:59:47.8788338Z +/lab3-valkey-1 2026-11-04T09:59:47.898667805Z + +$ make test-persistence PYTHON=python BASE_URL=http://localhost:28080 ACTION_SCRIPT=scripts/restart.cmd +python tests/resilience.py --mode persistence --action-script "scripts/restart.cmd" --base-url "http://localhost:28080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# StartedAt after: +/lab3-api-1 2026-11-04T10:01:10.565868768Z +/lab3-api-2 2026-11-04T10:01:10.940982484Z +/lab3-lb-1 2026-11-04T10:01:03.947016392Z +/lab3-postgres-1 2026-11-04T10:01:03.921315574Z +/lab3-valkey-1 2026-11-04T10:01:03.935862066Z diff --git a/evidence/rollout/instances-after.txt b/evidence/rollout/instances-after.txt new file mode 100644 index 00000000..b43e9b0f --- /dev/null +++ b/evidence/rollout/instances-after.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +96701b17071e lab3-api-5 running 77f32b420e9cde7a 74f0bfa45427 +0e9e0f7c8e70 lab3-api-6 running 418e8d8655cb156f 74f0bfa45427 diff --git a/evidence/rollout/instances-before.txt b/evidence/rollout/instances-before.txt new file mode 100644 index 00000000..13dc7cfa --- /dev/null +++ b/evidence/rollout/instances-before.txt @@ -0,0 +1,3 @@ +CONTAINER NAME STATE X-INSTANCE-ID IMAGE +5bcf254ae6ba lab3-api-3 running 6e898c3a3552ad08 74f0bfa45427 +c6c9938a148f lab3-api-4 running 0b71c96e22052a34 74f0bfa45427 diff --git a/evidence/rollout/meta.json b/evidence/rollout/meta.json new file mode 100644 index 00000000..1e0e42b1 --- /dev/null +++ b/evidence/rollout/meta.json @@ -0,0 +1,13 @@ +{ + "date_utc": "2026-11-04T10:14:34Z", + "commit": "1626145", + "branch": "solution/bpi2301-bakulin", + "docker": "28.5.1", + "host_cpus": "8", + "host_memory_bytes": "12547280896", + "experiment": "rolling-update", + "rate": "50", + "duration": "60s", + "update_at_s": "15", + "go": "go1.27.1" +} diff --git a/evidence/rollout/seconds.csv b/evidence/rollout/seconds.csv new file mode 100644 index 00000000..f3181d79 --- /dev/null +++ b/evidence/rollout/seconds.csv @@ -0,0 +1,61 @@ +second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances +0,50,50,0,0,0,0,0,0,3.8,4.8,7.1,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +1,50,50,0,0,0,0,0,0,3.8,4.8,7.4,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +2,50,50,0,0,0,0,0,0,3.4,5.6,6.5,2.2,0b71c96e22052a34=24|6e898c3a3552ad08=26 +3,50,50,0,0,0,0,0,0,3.1,5.1,8.3,1.1,0b71c96e22052a34=25|6e898c3a3552ad08=25 +4,50,50,0,0,0,0,0,0,3.2,4.6,5.4,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +5,50,50,0,0,0,0,0,0,3.1,4.5,8.3,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +6,50,50,0,0,0,0,0,0,3.4,4.7,7.3,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +7,50,50,0,0,0,0,0,0,3.6,4.5,5.8,1.3,0b71c96e22052a34=26|6e898c3a3552ad08=24 +8,50,50,0,0,0,0,0,0,3.8,5.0,5.5,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +9,50,50,0,0,0,0,0,0,4.7,8.2,11.3,1.6,0b71c96e22052a34=25|6e898c3a3552ad08=25 +10,50,50,0,0,0,0,0,0,6.7,9.7,12.9,1.4,0b71c96e22052a34=25|6e898c3a3552ad08=25 +11,50,50,0,0,0,0,0,0,3.7,5.6,6.2,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +12,50,50,0,0,0,0,0,0,3.2,4.7,4.9,1.5,0b71c96e22052a34=24|6e898c3a3552ad08=26 +13,50,50,0,0,0,0,0,0,3.3,4.5,7.3,1.1,0b71c96e22052a34=25|6e898c3a3552ad08=25 +14,50,50,0,0,0,0,0,0,3.6,4.9,9.1,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +15,50,50,0,0,0,0,0,0,3.8,5.1,6.0,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +16,50,50,0,0,0,0,0,0,4.2,5.5,6.5,1.2,0b71c96e22052a34=25|6e898c3a3552ad08=25 +17,50,50,0,0,0,0,0,0,4.4,5.8,7.4,1.4,0b71c96e22052a34=25|6e898c3a3552ad08=25 +18,50,50,0,0,0,0,0,0,3.9,6.7,8.3,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +19,50,50,0,0,0,0,0,0,3.9,6.0,6.3,1.3,0b71c96e22052a34=25|6e898c3a3552ad08=25 +20,50,50,0,0,0,0,0,0,4.6,8.0,10.9,1.2,0b71c96e22052a34=15|418e8d8655cb156f=10|6e898c3a3552ad08=16|77f32b420e9cde7a=9 +21,50,50,0,0,0,0,0,0,4.6,7.2,7.5,1.4,0b71c96e22052a34=13|418e8d8655cb156f=12|6e898c3a3552ad08=12|77f32b420e9cde7a=13 +22,50,50,0,0,0,0,0,0,4.0,6.0,6.8,1.3,0b71c96e22052a34=12|418e8d8655cb156f=13|6e898c3a3552ad08=13|77f32b420e9cde7a=12 +23,50,50,0,0,0,0,0,0,3.1,4.5,4.7,1.2,0b71c96e22052a34=12|418e8d8655cb156f=13|6e898c3a3552ad08=12|77f32b420e9cde7a=13 +24,50,50,0,0,0,0,0,0,4.4,8.3,13.9,1.3,0b71c96e22052a34=13|418e8d8655cb156f=12|6e898c3a3552ad08=13|77f32b420e9cde7a=12 +25,50,50,0,0,0,0,0,0,6.2,9.0,11.1,1.6,0b71c96e22052a34=12|418e8d8655cb156f=13|6e898c3a3552ad08=12|77f32b420e9cde7a=13 +26,50,50,0,0,0,0,0,0,3.7,4.9,5.4,1.2,0b71c96e22052a34=13|418e8d8655cb156f=12|6e898c3a3552ad08=13|77f32b420e9cde7a=12 +27,50,50,0,0,0,0,0,0,3.4,5.1,11.0,1.6,0b71c96e22052a34=5|418e8d8655cb156f=20|6e898c3a3552ad08=4|77f32b420e9cde7a=21 +28,50,50,0,0,0,0,0,0,3.6,6.5,15.9,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +29,50,50,0,0,0,0,0,0,3.8,6.0,7.7,1.4,418e8d8655cb156f=25|77f32b420e9cde7a=25 +30,50,50,0,0,0,0,0,0,3.7,4.9,7.6,1.6,418e8d8655cb156f=25|77f32b420e9cde7a=25 +31,50,50,0,0,0,0,0,0,3.7,4.8,6.9,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +32,50,50,0,0,0,0,0,0,3.6,4.6,6.7,1.8,418e8d8655cb156f=25|77f32b420e9cde7a=25 +33,50,50,0,0,0,0,0,0,3.1,4.2,5.2,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +34,50,50,0,0,0,0,0,0,3.2,4.5,9.8,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +35,50,50,0,0,0,0,0,0,3.2,5.3,6.0,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +36,50,50,0,0,0,0,0,0,3.1,4.4,5.6,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +37,50,50,0,0,0,0,0,0,3.2,4.2,6.3,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +38,50,50,0,0,0,0,0,0,3.8,6.5,7.4,10.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +39,50,50,0,0,0,0,0,0,5.2,12.8,31.3,1.4,418e8d8655cb156f=25|77f32b420e9cde7a=25 +40,50,50,0,0,0,0,0,0,6.6,9.4,12.0,4.4,418e8d8655cb156f=25|77f32b420e9cde7a=25 +41,50,50,0,0,0,0,0,0,3.8,5.4,7.1,1.6,418e8d8655cb156f=25|77f32b420e9cde7a=25 +42,50,50,0,0,0,0,0,0,3.0,4.0,4.4,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +43,50,50,0,0,0,0,0,0,3.0,4.2,4.3,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +44,50,50,0,0,0,0,0,0,3.9,5.5,6.0,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +45,50,50,0,0,0,0,0,0,3.9,5.4,6.0,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +46,50,50,0,0,0,0,0,0,3.3,4.8,5.3,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +47,50,50,0,0,0,0,0,0,3.1,4.6,6.8,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +48,50,50,0,0,0,0,0,0,3.4,6.2,7.3,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +49,50,50,0,0,0,0,0,0,3.3,5.2,5.6,1.6,418e8d8655cb156f=25|77f32b420e9cde7a=25 +50,50,50,0,0,0,0,0,0,3.2,5.0,6.5,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +51,50,50,0,0,0,0,0,0,3.6,4.7,6.2,1.7,418e8d8655cb156f=25|77f32b420e9cde7a=25 +52,50,50,0,0,0,0,0,0,3.0,4.2,6.6,1.3,418e8d8655cb156f=25|77f32b420e9cde7a=25 +53,50,50,0,0,0,0,0,0,2.9,4.1,4.4,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +54,50,50,0,0,0,0,0,0,3.1,5.0,5.6,1.5,418e8d8655cb156f=25|77f32b420e9cde7a=25 +55,50,50,0,0,0,0,0,0,3.3,4.3,6.9,3.5,418e8d8655cb156f=25|77f32b420e9cde7a=25 +56,50,50,0,0,0,0,0,0,3.4,5.0,5.4,1.4,418e8d8655cb156f=25|77f32b420e9cde7a=25 +57,50,50,0,0,0,0,0,0,3.1,4.7,5.0,3.6,418e8d8655cb156f=25|77f32b420e9cde7a=25 +58,50,50,0,0,0,0,0,0,3.4,5.0,6.2,1.2,418e8d8655cb156f=25|77f32b420e9cde7a=25 +59,50,50,0,0,0,0,0,0,3.0,4.5,7.4,1.9,418e8d8655cb156f=25|77f32b420e9cde7a=25 diff --git a/evidence/rollout/timeline.json b/evidence/rollout/timeline.json new file mode 100644 index 00000000..fe39695d --- /dev/null +++ b/evidence/rollout/timeline.json @@ -0,0 +1,45 @@ +{ + "started_at": "2026-11-04T10:14:38.29606054Z", + "config": { + "mode": "timeline", + "base_url": "http://lb:8080", + "users": 200, + "chats": 100, + "seed_messages_per_chat": 50, + "read_ratio": 0.8, + "rates": [ + 200, + 400 + ], + "step": "30s", + "warmup": "30s", + "rate": 50, + "duration": "60s", + "client_timeout": "2s", + "seed": 1, + "scrape": false, + "stop_on_fail": false, + "stats_url": "http://lb:8404/;csv", + "api_host": "api", + "debug_port": "9090" + }, + "messages_attempted": 3000, + "requests": 3000, + "confirmed": 3000, + "messages_retried": 0, + "unresolved": 0, + "rejected_4xx": 0, + "classes": { + "ok": 3000 + }, + "instances": { + "0b71c96e22052a34": 594, + "418e8d8655cb156f": 905, + "6e898c3a3552ad08": 596, + "77f32b420e9cde7a": 905 + }, + "history_messages": 3000, + "duplicates": 0, + "missing_confirmed": 0, + "passed": true +} diff --git a/evidence/rollout/update.json b/evidence/rollout/update.json new file mode 100644 index 00000000..6eb33db1 --- /dev/null +++ b/evidence/rollout/update.json @@ -0,0 +1,7 @@ +{ + "rollout_exit_code": 0, + "old_containers": "5bcf254ae6ba c6c9938a148f ", + "new_containers": [{"container": "96701b17071e", "started_vm_utc": "2026-11-04T10:14:58.25847985Z"}, {"container": "0e9e0f7c8e70", "started_vm_utc": "2026-11-04T10:14:57.97268484Z"}], + "command_started_host_utc": "2026-11-04T10:14:52.581Z", + "command_finished_host_utc": "2026-11-04T10:15:08.155Z" +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..416582bd --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,13 @@ +# Public checks against the stand +# date (UTC): 2026-11-04T10:00:01Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +$ make check PYTHON=python +python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +$ make test PYTHON=python BASE_URL=http://localhost:28080 +python tests/smoke.py --base-url "http://localhost:28080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + diff --git a/evidence/tests.txt b/evidence/tests.txt new file mode 100644 index 00000000..76d51e75 --- /dev/null +++ b/evidence/tests.txt @@ -0,0 +1,179 @@ +# go test -race -count=1 ./... (verbose output kept for the tests the report cites) +# date (UTC): 2026-11-04T10:07:19Z +# commit: f5f8040 on solution/bpi2301-bakulin +# host: Docker 28.5.1, 8 CPU, 12547280896 bytes RAM for the Linux VM; go1.27.1; Python 3.13.1 + +--- PASS: TestRunFailsFastWhenTheDebugListenerCannotStart (0.07s) +ok messenger/cmd/server 16.293s +--- PASS: TestHealthEndpoints (0.04s) +--- PASS: TestRegisterUserShapeAndConflicts (0.37s) +--- PASS: TestCurrentUserIsTheTokenOwner (0.43s) +--- PASS: TestChatCreationRules (0.47s) +--- PASS: TestChatVisibilityAndListing (0.65s) +--- PASS: TestChatListPagination (0.54s) +--- PASS: TestMessagesEndToEnd (0.72s) +--- PASS: TestMessageHistoryWalkAcrossPages (0.36s) +--- PASS: TestConcurrentSendsProduceExactlyUniqueMessages (1.45s) +--- PASS: TestUnknownRouteAndMethodAreContractErrors (0.02s) +--- PASS: TestLoginReturnsBearerSession (0.18s) +--- PASS: TestLoginFailuresAreUniformBasicChallenges (0.31s) +--- PASS: TestLoginAcceptsColonAndUnicodePasswords (0.12s) +--- PASS: TestProtectedRoutesAcceptOnlyBearer (0.22s) +--- PASS: TestLogoutRevokesOnlyTheCurrentToken (0.16s) +--- PASS: TestSessionExpiresAfterTTL (2.10s) +--- PASS: TestRegistrationIsPublicAndIgnoresAuthorization (0.05s) +--- PASS: TestValkeyOutageAnswers503 (4.57s) +--- PASS: TestPostgresOutageAnswers503 (4.27s) +--- PASS: TestHungPostgresAnswers503WithinTheRequestTimeout (5.67s) +--- PASS: TestSendIdempotencyKeyValidation (0.13s) +--- PASS: TestReplayReturnsIdenticalBody (0.12s) +--- PASS: TestReplayWithDifferentBodyIs409 (0.11s) +--- PASS: TestReplayRechecksSession (0.11s) +--- PASS: TestOutsiderWithoutKeyGets400AndWithKey404 (0.18s) +--- PASS: TestConcurrentReplaysAcrossTwoAppsCreateOneMessage (0.10s) +ok messenger/internal/app 39.344s +--- PASS: TestLoginIssuesOpaqueTokenBackedByStoredSession (0.28s) +--- PASS: TestLoginWithBadCredentialsCreatesNoSession (0.46s) +--- PASS: TestAuthenticateResolvesTheOwner (0.29s) +--- PASS: TestLogoutRevokesOnlyThatSession (0.32s) +--- PASS: TestSessionExpiresAfterTTL (1.32s) +ok messenger/internal/auth 19.269s +--- PASS: TestNewChatDerivesCreatedAtFromID (0.00s) +--- PASS: TestNewChatValidatesTitle (0.00s) +--- PASS: TestValidateMembersCopiesInCanonicalOrder (0.00s) +--- PASS: TestValidateMembersRejectsInvalidLists (0.00s) +--- PASS: TestNewMessageDerivesCreatedAt (0.00s) +--- PASS: TestNewMessageValidatesTextByCodePoints (0.00s) +--- PASS: TestNewMessageKeepsSurroundingWhitespace (0.00s) +--- PASS: TestSameContentComparesWhatTheSenderWrote (0.00s) +--- PASS: TestCreateChatChecksMembersInOneCall (0.49s) +--- PASS: TestCreateChatRejectsUnknownMember (0.32s) +--- PASS: TestCreateChatRejectsInvalidInputBeforeAnyLookup (0.45s) +--- PASS: TestGetHidesForeignAndUnknownChatsAlike (0.32s) +--- PASS: TestListPagesOnlyOwnChats (0.36s) +--- PASS: TestSendChecksAccessBeforeValidation (0.30s) +--- PASS: TestHistoryPagesAndHidesForeignChat (0.36s) +--- PASS: TestSendReplaysSameBody (0.38s) +--- PASS: TestSendConflictsOnDifferentBody (0.31s) +--- PASS: TestKeysAreScopedByUserAndChat (0.35s) +--- PASS: TestSendChecksAccessBeforeKey (0.34s) +--- PASS: TestInvalidTextWithUsedKeyIsInvalid (0.30s) +--- PASS: TestPurgeExpiredKeysDeletesInBatches (0.88s) +--- PASS: TestPurgeExpiredKeysWaitsAGraceAfterTTL (0.29s) +--- PASS: TestRunKeyCleanupPurgesAndStops (0.28s) +ok messenger/internal/chat 19.006s +--- PASS: TestAppendOnceInsertsThenReplays (0.18s) +--- PASS: TestAppendOnceTakesOverExpiredKey (0.14s) +--- PASS: TestAppendOnceWaitsForCleanerThenInserts (0.42s) +--- PASS: TestCleanerSkipsRowLockedByRequest (0.16s) +--- PASS: TestDeleteExpiredKeysOnlyExpiredAndUpToLimit (0.16s) +--- PASS: TestConcurrentAppendOnceSameKeyStoresOneMessage (0.35s) +--- PASS: TestChatCreateThenGetForMember (0.11s) +--- PASS: TestCreateRollsBackWhenMemberInsertFails (0.06s) +--- PASS: TestChatUnknownID (0.04s) +--- PASS: TestListByMemberFiltersOrdersAndPages (0.18s) +--- PASS: TestListByChatUnknownChatIsEmpty (0.03s) +--- PASS: TestAppendOutOfOrderIsListedByID (0.07s) +--- PASS: TestPagingWalksWholeHistoryWithoutGapsOrDuplicates (2.27s) +--- PASS: TestAfterUnknownIDStartsAtNextGreater (0.06s) +--- PASS: TestConcurrentAppendsKeepEveryMessage (0.12s) +ok messenger/internal/chat/store 17.694s +--- PASS: TestKindOfReturnsKindForApplicationError (0.00s) +--- PASS: TestKindOfSeesThroughWrapping (0.00s) +--- PASS: TestKindOfDefaultsToInternal (0.00s) +--- PASS: TestWrapKeepsCause (0.00s) +ok messenger/internal/platform/apperr 1.850s +--- PASS: TestDefaults (0.02s) +--- PASS: TestSecretFilesAreRequiredAndNeverEchoed (0.12s) +--- PASS: TestOverrides (0.03s) +--- PASS: TestInvalidValuesAreReported (0.16s) +--- PASS: TestLoadMigrateNeedsOnlyTheDatabaseURL (0.01s) +ok messenger/internal/platform/config 2.236s +--- PASS: TestEveryResponseCarriesInstanceID (0.08s) +--- PASS: TestUnknownRouteIsJSON404 (0.00s) +--- PASS: TestTrailingSlashIsNotRedirected (0.00s) +--- PASS: TestWrongMethodIsJSON405WithAllow (0.00s) +--- PASS: TestPanicBecomesJSON500WithoutDetails (0.00s) +--- PASS: TestForeignErrorBecomesJSON500WithoutDetails (0.00s) +--- PASS: TestApplicationErrorMapping (0.00s) +--- PASS: TestSuccessResponseHeadersAndEscaping (0.00s) +--- PASS: TestRequestIDFromTheBalancerIsKept (0.00s) +--- PASS: TestMalformedRequestIDIsReplaced (0.00s) +--- PASS: TestDecodeJSONHappyPath (0.00s) +--- PASS: TestDecodeJSONFailures (0.01s) +--- PASS: TestAccessLogHasRequestFieldsButNoBody (0.00s) +--- PASS: TestAccessLogLevelFollowsStatus (0.00s) +--- PASS: TestUserIDRoundTripAndAccessLogAttribution (0.00s) +--- PASS: TestWithUserPassesTheIdentifiedCaller (0.00s) +--- PASS: TestWithUserWithoutIdentityIsAWiringError (0.00s) +--- PASS: TestUnavailableCauseIsLoggedButNotReturned (0.00s) +--- PASS: TestTimeoutBoundsTheRequestContext (0.05s) +--- PASS: TestTimestampFormat (0.00s) +ok messenger/internal/platform/httpx 5.170s +--- PASS: TestNewProducesVersion7 (0.00s) +--- PASS: TestNewIsStrictlyIncreasingWithinProcess (0.08s) +--- PASS: TestTimeOfMatchesWallClockAtMillisecondPrecision (0.00s) +--- PASS: TestParseAcceptsCanonicalForm (0.00s) +--- PASS: TestParseRejectsNonCanonicalForms (0.00s) +--- PASS: TestNewAliasIs16HexAndRandom (0.00s) +ok messenger/internal/platform/ident 1.992s +--- PASS: TestCursorRoundTrip (0.00s) +--- PASS: TestDecodeCursorRejectsForeignScope (0.00s) +--- PASS: TestDecodeCursorRejectsGarbage (0.00s) +--- PASS: TestParseLimit (0.00s) +--- PASS: TestCutTrimsOverfetchAndDerivesNext (0.00s) +--- PASS: TestToResponseEncodesCursorWithinScope (0.00s) +--- PASS: TestToResponseSerializesEmptyPageAsContractRequires (0.00s) +ok messenger/internal/platform/pagination 2.129s +--- PASS: TestHashVerifiesOnlyTheOriginalPassword (0.67s) +--- PASS: TestHashIsSaltedPHCString (0.14s) +--- PASS: TestVerifyUsesParametersFromTheEncoding (0.03s) +--- PASS: TestVerifyRejectsMalformedEncoding (0.08s) +ok messenger/internal/platform/password 2.848s +--- PASS: TestPoolScansTimestamptzInUTC (0.01s) +--- PASS: TestMigrateIsRepeatable (0.05s) +--- PASS: TestPoolAppliesLimitsAndTimeouts (0.02s) +--- PASS: TestResetEmptiesEveryTableButKeepsSchemaVersion (0.12s) +--- PASS: TestIsUniqueViolation (0.06s) +--- PASS: TestClassifyMarksUnreachableDatabaseAsUnavailable (0.00s) +--- PASS: TestNewPoolUsesTheRequestedSize (0.08s) +--- PASS: TestClassifyRealConnectFailure (0.00s) +ok messenger/internal/platform/postgres 14.311s +--- PASS: TestNewClientPingsAndBoundsWaiting (4.01s) +--- PASS: TestNewClientErrorsNeverEchoTheURL (0.00s) +ok messenger/internal/platform/valkey 8.351s +--- PASS: TestServesUntilContextCancelledThenStopsCleanly (0.08s) +--- PASS: TestReadinessDropsBeforeListenerCloses (0.01s) +--- PASS: TestDrainKeepsServingWhileNotReady (0.71s) +--- PASS: TestRunDebugServesUntilCancelled (0.00s) +ok messenger/internal/server 2.039s +--- PASS: TestValkeyStartsAndResetFlushesKeys (4.05s) +ok messenger/internal/testenv 8.566s +--- PASS: TestRegisterStoresUserWithPasswordHash (0.18s) +--- PASS: TestRegisterRejectsInvalidInputAndStoresNothing (0.05s) +--- PASS: TestRegisterReportsDuplicateAsConflict (0.26s) +--- PASS: TestConcurrentRegistrationOfOneUsernameHasOneWinner (0.34s) +--- PASS: TestGetByIDUnknownIsNotFound (0.05s) +--- PASS: TestExistingIDsReturnsOnlyRegistered (0.24s) +--- PASS: TestAuthenticateChecksPassword (0.24s) +--- PASS: TestAuthenticateFailuresLookTheSame (0.66s) +--- PASS: TestNewDerivesCreatedAtFromID (0.00s) +--- PASS: TestNewValidatesUsername (0.00s) +--- PASS: TestValidatePasswordCountsCodePoints (0.00s) +--- PASS: TestNewValidatesDisplayNameByCodePoints (0.00s) +ok messenger/internal/user 14.322s +--- PASS: TestRunAgainstLiveServer (2.42s) +--- PASS: TestIdempotencyRequiresTwoReplicasByDefault (0.52s) +--- PASS: TestRunFailsOnUnreachableServer (0.00s) +ok messenger/tools/concurrency 17.499s +--- PASS: TestBenchAgainstLiveServer (2.46s) +--- PASS: TestTimelineVerifiesHistory (1.13s) +--- PASS: TestClassify (0.00s) +--- PASS: TestPercentileNearestRank (0.00s) +--- PASS: TestStepSummaryValidityAndErrorShare (0.00s) +--- PASS: TestOpenLoopDoesNotWaitForResponses (1.18s) +--- PASS: TestStepPassesOnlyWithinTheObjective (0.00s) +--- PASS: TestOnlyTheSystemEndsTheLadder (0.00s) +ok messenger/tools/loadgen 17.393s + +full suite: 18 packages ok, 0 failed -- 2.54.0 From 25f9564fd78ea07df25af776b00e9678030bfc58 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 13:36:26 +0300 Subject: [PATCH 13/21] =?UTF-8?q?=D0=94=D0=B0=D1=82=D1=8C=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B4=D0=B3=D0=BE=D1=82=D0=BE=D0=B2=D0=BA=D0=B5=20=D0=B4=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D1=8B=D1=85=20=D0=B2=20=D0=B3=D0=B5=D0=BD=D0=B5?= =?UTF-8?q?=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=B5=20=D0=BD=D0=B0=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B7=D0=BA=D0=B8=20=D0=BE=D1=82=D0=B4=D0=B5=D0=BB=D1=8C?= =?UTF-8?q?=D0=BD=D1=8B=D0=B9=20=D0=B4=D0=BB=D0=B8=D0=BD=D0=BD=D1=8B=D0=B9?= =?UTF-8?q?=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82:=20=D1=80=D0=B5?= =?UTF-8?q?=D0=B3=D0=B8=D1=81=D1=82=D1=80=D0=B0=D1=86=D0=B8=D1=8F=20=D1=81?= =?UTF-8?q?=20Argon2id=20=D0=BD=D0=B0=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8?= =?UTF-8?q?=D0=BA=D0=B5=20=D1=81=20=D0=BF=D0=BE=D0=BB=D0=BE=D0=B2=D0=B8?= =?UTF-8?q?=D0=BD=D0=BE=D0=B9=20CPU=20=D0=BD=D0=B5=20=D1=83=D0=BA=D0=BB?= =?UTF-8?q?=D0=B0=D0=B4=D1=8B=D0=B2=D0=B0=D0=B5=D1=82=D1=81=D1=8F=20=D0=B2?= =?UTF-8?q?=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82=20=D0=B8=D0=B7?= =?UTF-8?q?=D0=BC=D0=B5=D1=80=D0=B5=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/loadgen/main.go | 4 ++-- tools/loadgen/main_test.go | 33 ++++++++++++++++++++++++++++++++- tools/loadgen/seed.go | 5 +++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go index f081d981..2873bfee 100644 --- a/tools/loadgen/main.go +++ b/tools/loadgen/main.go @@ -158,7 +158,7 @@ type benchReport struct { func runBench(ctx context.Context, cfg config) error { c := newClient(cfg.BaseURL, cfg.timeout) rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) - ds, err := seed(ctx, c, rng, cfg.Users, cfg.Chats, cfg.SeedMessages) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages) if err != nil { return err } @@ -245,7 +245,7 @@ const retryGrace = 10 * time.Second func runTimeline(ctx context.Context, cfg config) error { c := newClient(cfg.BaseURL, cfg.timeout) rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) - ds, err := seed(ctx, c, rng, 2, 1, 0) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0) if err != nil { return err } diff --git a/tools/loadgen/main_test.go b/tools/loadgen/main_test.go index 78dba77c..4f6c8c75 100644 --- a/tools/loadgen/main_test.go +++ b/tools/loadgen/main_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "net/http/httptest" "os" "path/filepath" @@ -41,6 +42,10 @@ func TestMain(m *testing.M) { } func liveServer(t *testing.T) string { + return serve(t, nil) +} + +func serve(t *testing.T, wrap func(http.Handler) http.Handler) string { t.Helper() a := app.New(app.Deps{ Log: slog.New(slog.NewTextHandler(io.Discard, nil)), @@ -50,11 +55,37 @@ func liveServer(t *testing.T) string { MaxBodyBytes: 64 * 1024, InstanceID: "loadgen-test", }) - srv := httptest.NewServer(a.Engine) + var h http.Handler = a.Engine + if wrap != nil { + h = wrap(h) + } + srv := httptest.NewServer(h) t.Cleanup(srv.Close) return srv.URL } +// Registration hashes a password with Argon2id, which is slow on a small +// replica; the setup is not measured, so it must not fail on the short +// timeout of the measured requests. +func TestSlowSetupDoesNotFailTheRun(t *testing.T) { + slowAccounts := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && (r.URL.Path == "/api/v1/users" || r.URL.Path == "/api/v1/auth/sessions") { + time.Sleep(300 * time.Millisecond) + } + next.ServeHTTP(w, r) + }) + } + cfg, err := parse("bench", []string{"-base-url", serve(t, slowAccounts), "-users", "2", "-chats", "1", "-seed-messages", "1", + "-rates", "10", "-step", "1s", "-warmup", "0s", "-timeout", "100ms", "-out", t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatalf("a setup slower than -timeout failed the run: %v", err) + } +} + func TestBenchAgainstLiveServer(t *testing.T) { out := t.TempDir() cfg, err := parse("bench", []string{"-base-url", liveServer(t), "-users", "4", "-chats", "2", "-seed-messages", "3", diff --git a/tools/loadgen/seed.go b/tools/loadgen/seed.go index 748b0c18..61624236 100644 --- a/tools/loadgen/seed.go +++ b/tools/loadgen/seed.go @@ -7,6 +7,7 @@ import ( "fmt" mrand "math/rand/v2" "sync" + "time" "github.com/google/uuid" ) @@ -14,6 +15,10 @@ import ( // seedPassword is shared by the synthetic users of one run. const seedPassword = "loadgen-synthetic-password" +// seedTimeout bounds one setup request. The setup is not measured, and +// registration hashes a password with Argon2id, slow on a small replica. +const seedTimeout = 30 * time.Second + // seedWorkers bounds the parallel registrations: each one hashes a password // with Argon2id, and the setup must not be what the study measures. const seedWorkers = 8 -- 2.54.0 From 398f5f25e0ff11e36d3c2e12073da24ab451d260 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 15:26:09 +0300 Subject: [PATCH 14/21] =?UTF-8?q?=D0=A1=D0=B4=D0=B5=D0=BB=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=BE=D0=BB=D1=8C=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5=D1=80=D0=B8=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D1=82=20=D0=BF=D0=B0=D1=80=D0=BD=D1=8B=D0=BC:=20A2?= =?UTF-8?q?=20=D0=B8=20A2=20=D1=81=20=D1=83=D0=B4=D0=B2=D0=BE=D0=B5=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=BC=20CPU=20=D0=B1=D0=B0=D0=B7=D1=8B=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=BE=D1=87=D0=B5=D1=80=D1=91=D0=B4=D0=BD=D0=BE,=20?= =?UTF-8?q?=D0=B1=D0=B5=D0=B7=20=D0=BE=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=BB=D0=B5=D1=81=D1=82=D0=BD=D0=B8=D1=86=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/bench/series/control.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/bench/series/control.conf b/scripts/bench/series/control.conf index d00c775c..76a08eec 100644 --- a/scripts/bench/series/control.conf +++ b/scripts/bench/series/control.conf @@ -1,4 +1,6 @@ -# Control: A2 with twice the database CPU. If the database is the bottleneck, -# the capacity rises. +# Control: A2 next to A2 with twice the database CPU, interleaved, run with +# STOP_ON_FAIL=0 so that one noisy step does not end a ladder. If the database +# is the bottleneck, the capacity rises. # name replicas cpus memory gomemlimit pool maxconn pg_cpus +A2 2 1.0 512M 460MiB 10 0 1.0 A2pg2 2 1.0 512M 460MiB 10 0 2.0 -- 2.54.0 From 940bc6f9f76a232ec8603c46ffcae47030c29442 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 15:27:00 +0300 Subject: [PATCH 15/21] =?UTF-8?q?=D0=A1=D1=87=D0=B8=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B2=20=D0=BE=D1=82=D1=87=D1=91=D1=82=D0=B5=20=D1=8D?= =?UTF-8?q?=D0=BA=D1=81=D0=BF=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=BE=D0=B2=20CPU=20=D0=BD=D0=B0=20=D0=B7=D0=B0=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D1=81=20=D0=BF=D0=BE=20=D0=BA=D0=BE=D0=BC=D0=BF=D0=BE?= =?UTF-8?q?=D0=BD=D0=B5=D0=BD=D1=82=D0=B0=D0=BC=20=D0=B8=20=D0=BF=D0=BE?= =?UTF-8?q?=D1=82=D0=BE=D0=BB=D0=BE=D0=BA=20=D0=B1=D0=B0=D0=B7=D1=8B=20?= =?UTF-8?q?=D0=BF=D1=80=D0=B8=20=D1=82=D0=B0=D0=BA=D0=BE=D0=B9=20=D1=86?= =?UTF-8?q?=D0=B5=D0=BD=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/bench/plot.py | 50 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index cced9925..eb4017d6 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -10,8 +10,11 @@ output directory: control.csv the noise checks of every run, the host speed probe and the loadgen CPU per request at the first step, each flagged when more than 25 % off its median; + cost.csv CPU milliseconds per request of each component at the + COST_RATES steps (valid, error-free, unflagged runs only), and + the rate one database CPU would serve at that cost; -throughput.png, -p99.png, capacity.png, - cpu-at-capacity.png, pool-waits.png. + cpu-at-capacity.png, pool-waits.png, cost-per-request.png. Capacity is the highest step that passed the criterion loadgen applies (valid, under 1 % errors, p99 of successful requests within 500 ms). Charts show the median of the repeats; a hollow marker means the generator fell behind. @@ -30,6 +33,9 @@ import matplotlib.pyplot as plt # noqa: E402 SLO_P99_MS = 500.0 CONTROL_TOLERANCE = 0.25 +# Below saturation in every configuration, so the cost is not inflated by +# queueing. +COST_RATES = (800, 1000) COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") @@ -185,11 +191,51 @@ def main(root, out): w = csv.writer(f) w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) + flagged = set() for series, config, rep, probe, cpu in control: probe_dev = probe / probe_median - 1 if probe is not None else 0.0 cpu_dev = cpu / cpu_median - 1 + noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE + if noisy: + flagged.add((series, config, rep)) w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, - f"{cpu_dev:+.2f}", max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE]) + f"{cpu_dev:+.2f}", noisy]) + + cost = {} + for r in rows: + if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES: + continue + if not r["valid"] or r["error_share"] > 0: + continue + per = cost.setdefault((r["series"], r["config"]), {"runs": set()}) + per["runs"].add(r["rep"]) + for c in ("api", "postgres", "lb"): + if r[f"cpu_{c}_pct"] != "": + per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"]) + with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request", + "lb_cpu_ms_per_request", "rps_per_database_cpu"]) + for (series, config), per in cost.items(): + med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)} + w.writerow([series, config, " ".join(sorted(per["runs"])), + *(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")), + round(1000 / med["postgres"]) if med.get("postgres") else ""]) + if cost: + keys = list(cost) + fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5)) + width = 0.27 + for i, c in enumerate(("api", "postgres", "lb")): + ax.bar([j + i * width for j in range(len(keys))], + [statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c) + ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30) + ax.set_ylabel("CPU ms per request") + ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cost-per-request.png", dpi=120) + plt.close(fig) def median_by_offered(series, config, field): points, valid = {}, {} -- 2.54.0 From ecf89efc3984be18328212838ae0f39ef97b62e0 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 17:24:56 +0300 Subject: [PATCH 16/21] =?UTF-8?q?=D0=A1=D0=BE=D1=85=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D1=8F=D1=82=D1=8C=20=D0=B8=D0=B7=20=D0=BB=D0=BE=D0=B3=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=BF=D1=80=D0=BE=D0=B3=D0=BE=D0=BD=D0=B0=20=D1=82?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=81=D0=BE=D1=81=D1=82=D0=BE=D1=8F?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20=D0=B1=D0=B0=D0=BB=D0=B0=D0=BD=D1=81=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D1=89=D0=B8=D0=BA=D0=B0=20=D0=B8=20=D1=80?= =?UTF-8?q?=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA,=20=D1=81=D0=B6=D0=B8=D0=BC?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BF=D0=BE=D1=81=D0=B5=D0=BA=D1=83=D0=BD?= =?UTF-8?q?=D0=B4=D0=BD=D1=8B=D0=B5=20=D0=BC=D0=B5=D1=82=D1=80=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/bench/lib.sh | 9 +++++++++ scripts/bench/plot.py | 10 ++++++++-- scripts/bench/run-series.sh | 3 ++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index df5e37fc..4ddbe67b 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -38,6 +38,15 @@ host_probe() { -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' } +# state_log keeps from `compose logs` (stdin) what explains a run: the +# balancer's server state changes and the replicas' start and stop. An +# overloaded run logs every failed request, gigabytes over a series; those +# lines are only counted. +state_log() { + awk '/is DOWN|is UP|UP\/READY|maintenance|"msg":"(starting|shutdown started|shutdown completed)"/ { print; next } + { dropped++ } END { printf "# %d other lines dropped\n", dropped }' +} + # sample_docker_stats appends the CPU and memory of every container until killed. sample_docker_stats() { while :; do diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index eb4017d6..5258974c 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -20,6 +20,7 @@ under 1 % errors, p99 of successful requests within 500 ms). Charts show the median of the repeats; a hollow marker means the generator fell behind. """ import csv +import gzip import json import pathlib import statistics @@ -40,10 +41,15 @@ COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") def read_jsonl(path): - if not path.exists(): + packed = path.with_name(path.name + ".gz") + if packed.exists(): + text = gzip.decompress(packed.read_bytes()).decode("utf-8") + elif path.exists(): + text = path.read_text(encoding="utf-8") + else: return [] rows = [] - for line in path.read_text(encoding="utf-8").splitlines(): + for line in text.splitlines(): line = line.strip() if line: try: diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index f8b8de71..f291fa77 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -58,7 +58,8 @@ run_config() { echo " loadgen failed, see $dir/loadgen.log" kill "$stats" "$pg" 2>/dev/null || true wait "$stats" "$pg" 2>/dev/null || true - docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1 + docker compose logs --no-color --timestamps lb api 2>&1 | state_log >"$dir/compose.log" + gzip -9 "$dir"/*.jsonl grep '^step' "$dir/loadgen.log" || true } -- 2.54.0 From f447cfa08f1a903709cecea38bf7a76c9a481150 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 17:29:28 +0300 Subject: [PATCH 17/21] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=80=D0=B5=D0=B7=D1=83=D0=BB=D1=8C=D1=82=D0=B0?= =?UTF-8?q?=D1=82=D1=8B=20=D1=82=D1=80=D0=B5=D0=BA=D0=B0=201=20=C2=AB?= =?UTF-8?q?=D0=A6=D0=B5=D0=BD=D0=B0=20=D0=BC=D0=B0=D1=81=D1=88=D1=82=D0=B0?= =?UTF-8?q?=D0=B1=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=C2=BB=20?= =?UTF-8?q?=D0=B8=20=D0=BE=D1=82=D1=87=D1=91=D1=82:=20=D1=81=D0=B5=D1=80?= =?UTF-8?q?=D0=B8=D0=B8=20A,=20A=E2=80=B2,=20B,=20=D0=BF=D0=B0=D1=80=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=20=D1=81=20=D1=83=D0=B4=D0=B2=D0=BE=D0=B5=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D0=BC=20CPU=20=D0=B1=D0=B0=D0=B7=D1=8B,=20=D1=83=D0=BB=D1=83?= =?UTF-8?q?=D1=87=D1=88=D0=B5=D0=BD=D0=B8=D0=B5=20C=20=D1=81=20=D0=BE?= =?UTF-8?q?=D0=B3=D1=80=D0=B0=D0=BD=D0=B8=D1=87=D0=B5=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=D0=BC=20=D0=B7=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D0=BE=D0=B2=20?= =?UTF-8?q?=D0=BD=D0=B0=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/pull_request_template.md | 28 +- REPORT.md | 335 ++++++++++++++++- evidence/track1/A-p99.png | Bin 0 -> 63782 bytes evidence/track1/A-throughput.png | Bin 0 -> 49668 bytes evidence/track1/Aprime-p99.png | Bin 0 -> 52909 bytes evidence/track1/Aprime-throughput.png | Bin 0 -> 49436 bytes evidence/track1/B-p99.png | Bin 0 -> 65841 bytes evidence/track1/B-throughput.png | Bin 0 -> 55331 bytes evidence/track1/C-p99.png | Bin 0 -> 75645 bytes evidence/track1/C-throughput.png | Bin 0 -> 71123 bytes evidence/track1/capacity.csv | 17 + evidence/track1/capacity.png | Bin 0 -> 50331 bytes evidence/track1/control-p99.png | Bin 0 -> 51745 bytes evidence/track1/control-throughput.png | Bin 0 -> 55997 bytes evidence/track1/control.csv | 49 +++ evidence/track1/cost-per-request.png | Bin 0 -> 46901 bytes evidence/track1/cost.csv | 16 + evidence/track1/cpu-at-capacity.png | Bin 0 -> 55210 bytes evidence/track1/pool-waits.png | Bin 0 -> 129087 bytes .../raw/A/A1/rep1/docker-stats.jsonl.gz | Bin 0 -> 2627 bytes evidence/track1/raw/A/A1/rep1/meta.json | 27 ++ .../track1/raw/A/A1/rep1/metrics.jsonl.gz | Bin 0 -> 11000 bytes .../track1/raw/A/A1/rep1/postgres.jsonl.gz | Bin 0 -> 2601 bytes evidence/track1/raw/A/A1/rep1/seconds.csv | 152 ++++++++ evidence/track1/raw/A/A1/rep1/steps.json | 128 +++++++ .../raw/A/A1/rep2/docker-stats.jsonl.gz | Bin 0 -> 3497 bytes evidence/track1/raw/A/A1/rep2/meta.json | 27 ++ .../track1/raw/A/A1/rep2/metrics.jsonl.gz | Bin 0 -> 15756 bytes .../track1/raw/A/A1/rep2/postgres.jsonl.gz | Bin 0 -> 3666 bytes evidence/track1/raw/A/A1/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/A/A1/rep2/steps.json | 172 +++++++++ .../raw/A/A1/rep3/docker-stats.jsonl.gz | Bin 0 -> 2617 bytes evidence/track1/raw/A/A1/rep3/meta.json | 27 ++ .../track1/raw/A/A1/rep3/metrics.jsonl.gz | Bin 0 -> 10983 bytes .../track1/raw/A/A1/rep3/postgres.jsonl.gz | Bin 0 -> 2613 bytes evidence/track1/raw/A/A1/rep3/seconds.csv | 152 ++++++++ evidence/track1/raw/A/A1/rep3/steps.json | 128 +++++++ .../raw/A/A2/rep1/docker-stats.jsonl.gz | Bin 0 -> 4332 bytes evidence/track1/raw/A/A2/rep1/meta.json | 27 ++ .../track1/raw/A/A2/rep1/metrics.jsonl.gz | Bin 0 -> 27125 bytes .../track1/raw/A/A2/rep1/postgres.jsonl.gz | Bin 0 -> 3640 bytes evidence/track1/raw/A/A2/rep1/seconds.csv | 212 +++++++++++ evidence/track1/raw/A/A2/rep1/steps.json | 178 +++++++++ .../raw/A/A2/rep2/docker-stats.jsonl.gz | Bin 0 -> 3720 bytes evidence/track1/raw/A/A2/rep2/meta.json | 27 ++ .../track1/raw/A/A2/rep2/metrics.jsonl.gz | Bin 0 -> 21735 bytes .../track1/raw/A/A2/rep2/postgres.jsonl.gz | Bin 0 -> 3240 bytes evidence/track1/raw/A/A2/rep2/seconds.csv | 182 ++++++++++ evidence/track1/raw/A/A2/rep2/steps.json | 155 ++++++++ .../raw/A/A2/rep3/docker-stats.jsonl.gz | Bin 0 -> 3247 bytes evidence/track1/raw/A/A2/rep3/meta.json | 27 ++ .../track1/raw/A/A2/rep3/metrics.jsonl.gz | Bin 0 -> 18245 bytes .../track1/raw/A/A2/rep3/postgres.jsonl.gz | Bin 0 -> 2677 bytes evidence/track1/raw/A/A2/rep3/seconds.csv | 152 ++++++++ evidence/track1/raw/A/A2/rep3/steps.json | 132 +++++++ .../raw/A/A4/rep1/docker-stats.jsonl.gz | Bin 0 -> 3434 bytes evidence/track1/raw/A/A4/rep1/meta.json | 27 ++ .../track1/raw/A/A4/rep1/metrics.jsonl.gz | Bin 0 -> 24233 bytes .../track1/raw/A/A4/rep1/postgres.jsonl.gz | Bin 0 -> 2270 bytes evidence/track1/raw/A/A4/rep1/seconds.csv | 122 +++++++ evidence/track1/raw/A/A4/rep1/steps.json | 115 ++++++ .../raw/A/A4/rep2/docker-stats.jsonl.gz | Bin 0 -> 5559 bytes evidence/track1/raw/A/A4/rep2/meta.json | 27 ++ .../track1/raw/A/A4/rep2/metrics.jsonl.gz | Bin 0 -> 46808 bytes .../track1/raw/A/A4/rep2/postgres.jsonl.gz | Bin 0 -> 3658 bytes evidence/track1/raw/A/A4/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/A/A4/rep2/steps.json | 190 ++++++++++ .../raw/A/A4/rep3/docker-stats.jsonl.gz | Bin 0 -> 4106 bytes evidence/track1/raw/A/A4/rep3/meta.json | 27 ++ .../track1/raw/A/A4/rep3/metrics.jsonl.gz | Bin 0 -> 31995 bytes .../track1/raw/A/A4/rep3/postgres.jsonl.gz | Bin 0 -> 2689 bytes evidence/track1/raw/A/A4/rep3/seconds.csv | 152 ++++++++ evidence/track1/raw/A/A4/rep3/steps.json | 140 ++++++++ .../raw/Aprime/Ap1/rep1/docker-stats.jsonl.gz | Bin 0 -> 3039 bytes evidence/track1/raw/Aprime/Ap1/rep1/meta.json | 27 ++ .../raw/Aprime/Ap1/rep1/metrics.jsonl.gz | Bin 0 -> 12971 bytes .../raw/Aprime/Ap1/rep1/postgres.jsonl.gz | Bin 0 -> 3194 bytes .../track1/raw/Aprime/Ap1/rep1/seconds.csv | 182 ++++++++++ .../track1/raw/Aprime/Ap1/rep1/steps.json | 150 ++++++++ .../raw/Aprime/Ap1/rep2/docker-stats.jsonl.gz | Bin 0 -> 2626 bytes evidence/track1/raw/Aprime/Ap1/rep2/meta.json | 27 ++ .../raw/Aprime/Ap1/rep2/metrics.jsonl.gz | Bin 0 -> 10750 bytes .../raw/Aprime/Ap1/rep2/postgres.jsonl.gz | Bin 0 -> 2704 bytes .../track1/raw/Aprime/Ap1/rep2/seconds.csv | 152 ++++++++ .../track1/raw/Aprime/Ap1/rep2/steps.json | 128 +++++++ .../raw/Aprime/Ap1/rep3/docker-stats.jsonl.gz | Bin 0 -> 1700 bytes evidence/track1/raw/Aprime/Ap1/rep3/meta.json | 27 ++ .../raw/Aprime/Ap1/rep3/metrics.jsonl.gz | Bin 0 -> 6996 bytes .../raw/Aprime/Ap1/rep3/postgres.jsonl.gz | Bin 0 -> 1705 bytes .../track1/raw/Aprime/Ap1/rep3/seconds.csv | 92 +++++ .../track1/raw/Aprime/Ap1/rep3/steps.json | 84 +++++ .../raw/Aprime/Ap4/rep1/docker-stats.jsonl.gz | Bin 0 -> 5427 bytes evidence/track1/raw/Aprime/Ap4/rep1/meta.json | 27 ++ .../raw/Aprime/Ap4/rep1/metrics.jsonl.gz | Bin 0 -> 47082 bytes .../raw/Aprime/Ap4/rep1/postgres.jsonl.gz | Bin 0 -> 3687 bytes .../track1/raw/Aprime/Ap4/rep1/seconds.csv | 212 +++++++++++ .../track1/raw/Aprime/Ap4/rep1/steps.json | 190 ++++++++++ .../raw/Aprime/Ap4/rep2/docker-stats.jsonl.gz | Bin 0 -> 4178 bytes evidence/track1/raw/Aprime/Ap4/rep2/meta.json | 27 ++ .../raw/Aprime/Ap4/rep2/metrics.jsonl.gz | Bin 0 -> 33437 bytes .../raw/Aprime/Ap4/rep2/postgres.jsonl.gz | Bin 0 -> 2607 bytes .../track1/raw/Aprime/Ap4/rep2/seconds.csv | 152 ++++++++ .../track1/raw/Aprime/Ap4/rep2/steps.json | 140 ++++++++ .../raw/Aprime/Ap4/rep3/docker-stats.jsonl.gz | Bin 0 -> 2794 bytes evidence/track1/raw/Aprime/Ap4/rep3/meta.json | 27 ++ .../raw/Aprime/Ap4/rep3/metrics.jsonl.gz | Bin 0 -> 20559 bytes .../raw/Aprime/Ap4/rep3/postgres.jsonl.gz | Bin 0 -> 1666 bytes .../track1/raw/Aprime/Ap4/rep3/seconds.csv | 92 +++++ .../track1/raw/Aprime/Ap4/rep3/steps.json | 90 +++++ .../raw/B/B1/rep1/docker-stats.jsonl.gz | Bin 0 -> 2775 bytes evidence/track1/raw/B/B1/rep1/meta.json | 27 ++ .../track1/raw/B/B1/rep1/metrics.jsonl.gz | Bin 0 -> 11134 bytes .../track1/raw/B/B1/rep1/postgres.jsonl.gz | Bin 0 -> 2808 bytes evidence/track1/raw/B/B1/rep1/seconds.csv | 152 ++++++++ evidence/track1/raw/B/B1/rep1/steps.json | 128 +++++++ .../raw/B/B1/rep2/docker-stats.jsonl.gz | Bin 0 -> 1860 bytes evidence/track1/raw/B/B1/rep2/meta.json | 27 ++ .../track1/raw/B/B1/rep2/metrics.jsonl.gz | Bin 0 -> 6734 bytes .../track1/raw/B/B1/rep2/postgres.jsonl.gz | Bin 0 -> 1837 bytes evidence/track1/raw/B/B1/rep2/seconds.csv | 92 +++++ evidence/track1/raw/B/B1/rep2/steps.json | 86 +++++ .../raw/B/B1/rep3/docker-stats.jsonl.gz | Bin 0 -> 3194 bytes evidence/track1/raw/B/B1/rep3/meta.json | 27 ++ .../track1/raw/B/B1/rep3/metrics.jsonl.gz | Bin 0 -> 13614 bytes .../track1/raw/B/B1/rep3/postgres.jsonl.gz | Bin 0 -> 3326 bytes evidence/track1/raw/B/B1/rep3/seconds.csv | 182 ++++++++++ evidence/track1/raw/B/B1/rep3/steps.json | 150 ++++++++ .../raw/B/B3/rep1/docker-stats.jsonl.gz | Bin 0 -> 4191 bytes evidence/track1/raw/B/B3/rep1/meta.json | 27 ++ .../track1/raw/B/B3/rep1/metrics.jsonl.gz | Bin 0 -> 30553 bytes .../track1/raw/B/B3/rep1/postgres.jsonl.gz | Bin 0 -> 3110 bytes evidence/track1/raw/B/B3/rep1/seconds.csv | 182 ++++++++++ evidence/track1/raw/B/B3/rep1/steps.json | 160 +++++++++ .../raw/B/B3/rep2/docker-stats.jsonl.gz | Bin 0 -> 2457 bytes evidence/track1/raw/B/B3/rep2/meta.json | 27 ++ .../track1/raw/B/B3/rep2/metrics.jsonl.gz | Bin 0 -> 14611 bytes .../track1/raw/B/B3/rep2/postgres.jsonl.gz | Bin 0 -> 1722 bytes evidence/track1/raw/B/B3/rep2/seconds.csv | 92 +++++ evidence/track1/raw/B/B3/rep2/steps.json | 88 +++++ .../raw/B/B3/rep3/docker-stats.jsonl.gz | Bin 0 -> 4803 bytes evidence/track1/raw/B/B3/rep3/meta.json | 27 ++ .../track1/raw/B/B3/rep3/metrics.jsonl.gz | Bin 0 -> 37898 bytes .../track1/raw/B/B3/rep3/postgres.jsonl.gz | Bin 0 -> 3562 bytes evidence/track1/raw/B/B3/rep3/seconds.csv | 212 +++++++++++ evidence/track1/raw/B/B3/rep3/steps.json | 185 ++++++++++ .../raw/B/B4/rep1/docker-stats.jsonl.gz | Bin 0 -> 5427 bytes evidence/track1/raw/B/B4/rep1/meta.json | 27 ++ .../track1/raw/B/B4/rep1/metrics.jsonl.gz | Bin 0 -> 46924 bytes .../track1/raw/B/B4/rep1/postgres.jsonl.gz | Bin 0 -> 3490 bytes evidence/track1/raw/B/B4/rep1/seconds.csv | 212 +++++++++++ evidence/track1/raw/B/B4/rep1/steps.json | 192 ++++++++++ .../raw/B/B4/rep2/docker-stats.jsonl.gz | Bin 0 -> 4059 bytes evidence/track1/raw/B/B4/rep2/meta.json | 27 ++ .../track1/raw/B/B4/rep2/metrics.jsonl.gz | Bin 0 -> 33925 bytes .../track1/raw/B/B4/rep2/postgres.jsonl.gz | Bin 0 -> 2454 bytes evidence/track1/raw/B/B4/rep2/seconds.csv | 152 ++++++++ evidence/track1/raw/B/B4/rep2/steps.json | 140 ++++++++ .../raw/B/B4/rep3/docker-stats.jsonl.gz | Bin 0 -> 5402 bytes evidence/track1/raw/B/B4/rep3/meta.json | 27 ++ .../track1/raw/B/B4/rep3/metrics.jsonl.gz | Bin 0 -> 47050 bytes .../track1/raw/B/B4/rep3/postgres.jsonl.gz | Bin 0 -> 3509 bytes evidence/track1/raw/B/B4/rep3/seconds.csv | 212 +++++++++++ evidence/track1/raw/B/B4/rep3/steps.json | 192 ++++++++++ .../raw/C/R1m0/rep1/docker-stats.jsonl.gz | Bin 0 -> 3778 bytes evidence/track1/raw/C/R1m0/rep1/meta.json | 27 ++ .../track1/raw/C/R1m0/rep1/metrics.jsonl.gz | Bin 0 -> 17860 bytes .../track1/raw/C/R1m0/rep1/postgres.jsonl.gz | Bin 0 -> 3465 bytes evidence/track1/raw/C/R1m0/rep1/seconds.csv | 216 +++++++++++ evidence/track1/raw/C/R1m0/rep1/steps.json | 176 +++++++++ .../raw/C/R1m0/rep2/docker-stats.jsonl.gz | Bin 0 -> 3702 bytes evidence/track1/raw/C/R1m0/rep2/meta.json | 27 ++ .../track1/raw/C/R1m0/rep2/metrics.jsonl.gz | Bin 0 -> 17805 bytes .../track1/raw/C/R1m0/rep2/postgres.jsonl.gz | Bin 0 -> 3438 bytes evidence/track1/raw/C/R1m0/rep2/seconds.csv | 216 +++++++++++ evidence/track1/raw/C/R1m0/rep2/steps.json | 176 +++++++++ .../raw/C/R1m0/rep3/docker-stats.jsonl.gz | Bin 0 -> 3781 bytes evidence/track1/raw/C/R1m0/rep3/meta.json | 27 ++ .../track1/raw/C/R1m0/rep3/metrics.jsonl.gz | Bin 0 -> 17848 bytes .../track1/raw/C/R1m0/rep3/postgres.jsonl.gz | Bin 0 -> 3647 bytes evidence/track1/raw/C/R1m0/rep3/seconds.csv | 214 +++++++++++ evidence/track1/raw/C/R1m0/rep3/steps.json | 174 +++++++++ .../raw/C/R1m10/rep1/docker-stats.jsonl.gz | Bin 0 -> 3700 bytes evidence/track1/raw/C/R1m10/rep1/meta.json | 27 ++ .../track1/raw/C/R1m10/rep1/metrics.jsonl.gz | Bin 0 -> 16006 bytes .../track1/raw/C/R1m10/rep1/postgres.jsonl.gz | Bin 0 -> 3674 bytes evidence/track1/raw/C/R1m10/rep1/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R1m10/rep1/steps.json | 172 +++++++++ .../raw/C/R1m10/rep2/docker-stats.jsonl.gz | Bin 0 -> 3716 bytes evidence/track1/raw/C/R1m10/rep2/meta.json | 27 ++ .../track1/raw/C/R1m10/rep2/metrics.jsonl.gz | Bin 0 -> 15894 bytes .../track1/raw/C/R1m10/rep2/postgres.jsonl.gz | Bin 0 -> 3763 bytes evidence/track1/raw/C/R1m10/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R1m10/rep2/steps.json | 172 +++++++++ .../raw/C/R1m10/rep3/docker-stats.jsonl.gz | Bin 0 -> 3639 bytes evidence/track1/raw/C/R1m10/rep3/meta.json | 27 ++ .../track1/raw/C/R1m10/rep3/metrics.jsonl.gz | Bin 0 -> 16314 bytes .../track1/raw/C/R1m10/rep3/postgres.jsonl.gz | Bin 0 -> 3538 bytes evidence/track1/raw/C/R1m10/rep3/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R1m10/rep3/steps.json | 172 +++++++++ .../raw/C/R1m20/rep1/docker-stats.jsonl.gz | Bin 0 -> 3623 bytes evidence/track1/raw/C/R1m20/rep1/meta.json | 27 ++ .../track1/raw/C/R1m20/rep1/metrics.jsonl.gz | Bin 0 -> 18355 bytes .../track1/raw/C/R1m20/rep1/postgres.jsonl.gz | Bin 0 -> 3468 bytes evidence/track1/raw/C/R1m20/rep1/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R1m20/rep1/steps.json | 172 +++++++++ .../raw/C/R1m20/rep2/docker-stats.jsonl.gz | Bin 0 -> 3731 bytes evidence/track1/raw/C/R1m20/rep2/meta.json | 27 ++ .../track1/raw/C/R1m20/rep2/metrics.jsonl.gz | Bin 0 -> 17377 bytes .../track1/raw/C/R1m20/rep2/postgres.jsonl.gz | Bin 0 -> 3806 bytes evidence/track1/raw/C/R1m20/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R1m20/rep2/steps.json | 171 +++++++++ .../raw/C/R1m20/rep3/docker-stats.jsonl.gz | Bin 0 -> 3720 bytes evidence/track1/raw/C/R1m20/rep3/meta.json | 27 ++ .../track1/raw/C/R1m20/rep3/metrics.jsonl.gz | Bin 0 -> 17741 bytes .../track1/raw/C/R1m20/rep3/postgres.jsonl.gz | Bin 0 -> 3749 bytes evidence/track1/raw/C/R1m20/rep3/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R1m20/rep3/steps.json | 172 +++++++++ .../raw/C/R1m40/rep1/docker-stats.jsonl.gz | Bin 0 -> 3714 bytes evidence/track1/raw/C/R1m40/rep1/meta.json | 27 ++ .../track1/raw/C/R1m40/rep1/metrics.jsonl.gz | Bin 0 -> 17789 bytes .../track1/raw/C/R1m40/rep1/postgres.jsonl.gz | Bin 0 -> 3741 bytes evidence/track1/raw/C/R1m40/rep1/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R1m40/rep1/steps.json | 172 +++++++++ .../raw/C/R1m40/rep2/docker-stats.jsonl.gz | Bin 0 -> 3704 bytes evidence/track1/raw/C/R1m40/rep2/meta.json | 27 ++ .../track1/raw/C/R1m40/rep2/metrics.jsonl.gz | Bin 0 -> 17860 bytes .../track1/raw/C/R1m40/rep2/postgres.jsonl.gz | Bin 0 -> 3785 bytes evidence/track1/raw/C/R1m40/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R1m40/rep2/steps.json | 171 +++++++++ .../raw/C/R1m40/rep3/docker-stats.jsonl.gz | Bin 0 -> 3672 bytes evidence/track1/raw/C/R1m40/rep3/meta.json | 27 ++ .../track1/raw/C/R1m40/rep3/metrics.jsonl.gz | Bin 0 -> 18130 bytes .../track1/raw/C/R1m40/rep3/postgres.jsonl.gz | Bin 0 -> 3696 bytes evidence/track1/raw/C/R1m40/rep3/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R1m40/rep3/steps.json | 172 +++++++++ .../raw/C/R4m0/rep1/docker-stats.jsonl.gz | Bin 0 -> 5501 bytes evidence/track1/raw/C/R4m0/rep1/meta.json | 27 ++ .../track1/raw/C/R4m0/rep1/metrics.jsonl.gz | Bin 0 -> 51093 bytes .../track1/raw/C/R4m0/rep1/postgres.jsonl.gz | Bin 0 -> 3098 bytes evidence/track1/raw/C/R4m0/rep1/seconds.csv | 216 +++++++++++ evidence/track1/raw/C/R4m0/rep1/steps.json | 193 ++++++++++ .../raw/C/R4m0/rep2/docker-stats.jsonl.gz | Bin 0 -> 5586 bytes evidence/track1/raw/C/R4m0/rep2/meta.json | 27 ++ .../track1/raw/C/R4m0/rep2/metrics.jsonl.gz | Bin 0 -> 49366 bytes .../track1/raw/C/R4m0/rep2/postgres.jsonl.gz | Bin 0 -> 3478 bytes evidence/track1/raw/C/R4m0/rep2/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R4m0/rep2/steps.json | 191 ++++++++++ .../raw/C/R4m0/rep3/docker-stats.jsonl.gz | Bin 0 -> 5521 bytes evidence/track1/raw/C/R4m0/rep3/meta.json | 27 ++ .../track1/raw/C/R4m0/rep3/metrics.jsonl.gz | Bin 0 -> 51167 bytes .../track1/raw/C/R4m0/rep3/postgres.jsonl.gz | Bin 0 -> 3342 bytes evidence/track1/raw/C/R4m0/rep3/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R4m0/rep3/steps.json | 191 ++++++++++ .../raw/C/R4m20/rep1/docker-stats.jsonl.gz | Bin 0 -> 5152 bytes evidence/track1/raw/C/R4m20/rep1/meta.json | 27 ++ .../track1/raw/C/R4m20/rep1/metrics.jsonl.gz | Bin 0 -> 52307 bytes .../track1/raw/C/R4m20/rep1/postgres.jsonl.gz | Bin 0 -> 3204 bytes evidence/track1/raw/C/R4m20/rep1/seconds.csv | 213 +++++++++++ evidence/track1/raw/C/R4m20/rep1/steps.json | 190 ++++++++++ .../raw/C/R4m20/rep2/docker-stats.jsonl.gz | Bin 0 -> 5476 bytes evidence/track1/raw/C/R4m20/rep2/meta.json | 27 ++ .../track1/raw/C/R4m20/rep2/metrics.jsonl.gz | Bin 0 -> 49543 bytes .../track1/raw/C/R4m20/rep2/postgres.jsonl.gz | Bin 0 -> 3653 bytes evidence/track1/raw/C/R4m20/rep2/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R4m20/rep2/steps.json | 190 ++++++++++ .../raw/C/R4m20/rep3/docker-stats.jsonl.gz | Bin 0 -> 5323 bytes evidence/track1/raw/C/R4m20/rep3/meta.json | 27 ++ .../track1/raw/C/R4m20/rep3/metrics.jsonl.gz | Bin 0 -> 50024 bytes .../track1/raw/C/R4m20/rep3/postgres.jsonl.gz | Bin 0 -> 3566 bytes evidence/track1/raw/C/R4m20/rep3/seconds.csv | 212 +++++++++++ evidence/track1/raw/C/R4m20/rep3/steps.json | 190 ++++++++++ .../raw/control/A2/rep1/docker-stats.jsonl.gz | Bin 0 -> 6458 bytes evidence/track1/raw/control/A2/rep1/meta.json | 27 ++ .../raw/control/A2/rep1/metrics.jsonl.gz | Bin 0 -> 47072 bytes .../raw/control/A2/rep1/postgres.jsonl.gz | Bin 0 -> 5447 bytes .../track1/raw/control/A2/rep1/seconds.csv | 338 +++++++++++++++++ .../track1/raw/control/A2/rep1/steps.json | 278 ++++++++++++++ .../raw/control/A2/rep2/docker-stats.jsonl.gz | Bin 0 -> 6600 bytes evidence/track1/raw/control/A2/rep2/meta.json | 27 ++ .../raw/control/A2/rep2/metrics.jsonl.gz | Bin 0 -> 47043 bytes .../raw/control/A2/rep2/postgres.jsonl.gz | Bin 0 -> 5476 bytes .../track1/raw/control/A2/rep2/seconds.csv | 337 +++++++++++++++++ .../track1/raw/control/A2/rep2/steps.json | 278 ++++++++++++++ .../raw/control/A2/rep3/docker-stats.jsonl.gz | Bin 0 -> 6417 bytes evidence/track1/raw/control/A2/rep3/meta.json | 27 ++ .../raw/control/A2/rep3/metrics.jsonl.gz | Bin 0 -> 46917 bytes .../raw/control/A2/rep3/postgres.jsonl.gz | Bin 0 -> 5395 bytes .../track1/raw/control/A2/rep3/seconds.csv | 339 ++++++++++++++++++ .../track1/raw/control/A2/rep3/steps.json | 278 ++++++++++++++ .../control/A2pg2/rep1/docker-stats.jsonl.gz | Bin 0 -> 6504 bytes .../track1/raw/control/A2pg2/rep1/meta.json | 27 ++ .../raw/control/A2pg2/rep1/metrics.jsonl.gz | Bin 0 -> 44502 bytes .../raw/control/A2pg2/rep1/postgres.jsonl.gz | Bin 0 -> 5410 bytes .../track1/raw/control/A2pg2/rep1/seconds.csv | 335 +++++++++++++++++ .../track1/raw/control/A2pg2/rep1/steps.json | 274 ++++++++++++++ .../control/A2pg2/rep2/docker-stats.jsonl.gz | Bin 0 -> 6223 bytes .../track1/raw/control/A2pg2/rep2/meta.json | 27 ++ .../raw/control/A2pg2/rep2/metrics.jsonl.gz | Bin 0 -> 43232 bytes .../raw/control/A2pg2/rep2/postgres.jsonl.gz | Bin 0 -> 5452 bytes .../track1/raw/control/A2pg2/rep2/seconds.csv | 332 +++++++++++++++++ .../track1/raw/control/A2pg2/rep2/steps.json | 273 ++++++++++++++ .../control/A2pg2/rep3/docker-stats.jsonl.gz | Bin 0 -> 6442 bytes .../track1/raw/control/A2pg2/rep3/meta.json | 27 ++ .../raw/control/A2pg2/rep3/metrics.jsonl.gz | Bin 0 -> 43936 bytes .../raw/control/A2pg2/rep3/postgres.jsonl.gz | Bin 0 -> 5574 bytes .../track1/raw/control/A2pg2/rep3/seconds.csv | 334 +++++++++++++++++ .../track1/raw/control/A2pg2/rep3/steps.json | 273 ++++++++++++++ evidence/track1/summary.csv | 274 ++++++++++++++ 308 files changed, 20068 insertions(+), 29 deletions(-) create mode 100644 evidence/track1/A-p99.png create mode 100644 evidence/track1/A-throughput.png create mode 100644 evidence/track1/Aprime-p99.png create mode 100644 evidence/track1/Aprime-throughput.png create mode 100644 evidence/track1/B-p99.png create mode 100644 evidence/track1/B-throughput.png create mode 100644 evidence/track1/C-p99.png create mode 100644 evidence/track1/C-throughput.png create mode 100644 evidence/track1/capacity.csv create mode 100644 evidence/track1/capacity.png create mode 100644 evidence/track1/control-p99.png create mode 100644 evidence/track1/control-throughput.png create mode 100644 evidence/track1/control.csv create mode 100644 evidence/track1/cost-per-request.png create mode 100644 evidence/track1/cost.csv create mode 100644 evidence/track1/cpu-at-capacity.png create mode 100644 evidence/track1/pool-waits.png create mode 100644 evidence/track1/raw/A/A1/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep1/meta.json create mode 100644 evidence/track1/raw/A/A1/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep1/seconds.csv create mode 100644 evidence/track1/raw/A/A1/rep1/steps.json create mode 100644 evidence/track1/raw/A/A1/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep2/meta.json create mode 100644 evidence/track1/raw/A/A1/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep2/seconds.csv create mode 100644 evidence/track1/raw/A/A1/rep2/steps.json create mode 100644 evidence/track1/raw/A/A1/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep3/meta.json create mode 100644 evidence/track1/raw/A/A1/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A1/rep3/seconds.csv create mode 100644 evidence/track1/raw/A/A1/rep3/steps.json create mode 100644 evidence/track1/raw/A/A2/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep1/meta.json create mode 100644 evidence/track1/raw/A/A2/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep1/seconds.csv create mode 100644 evidence/track1/raw/A/A2/rep1/steps.json create mode 100644 evidence/track1/raw/A/A2/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep2/meta.json create mode 100644 evidence/track1/raw/A/A2/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep2/seconds.csv create mode 100644 evidence/track1/raw/A/A2/rep2/steps.json create mode 100644 evidence/track1/raw/A/A2/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep3/meta.json create mode 100644 evidence/track1/raw/A/A2/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A2/rep3/seconds.csv create mode 100644 evidence/track1/raw/A/A2/rep3/steps.json create mode 100644 evidence/track1/raw/A/A4/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep1/meta.json create mode 100644 evidence/track1/raw/A/A4/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep1/seconds.csv create mode 100644 evidence/track1/raw/A/A4/rep1/steps.json create mode 100644 evidence/track1/raw/A/A4/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep2/meta.json create mode 100644 evidence/track1/raw/A/A4/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep2/seconds.csv create mode 100644 evidence/track1/raw/A/A4/rep2/steps.json create mode 100644 evidence/track1/raw/A/A4/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep3/meta.json create mode 100644 evidence/track1/raw/A/A4/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/A/A4/rep3/seconds.csv create mode 100644 evidence/track1/raw/A/A4/rep3/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap1/rep1/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap1/rep2/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap1/rep3/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap4/rep1/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap4/rep2/steps.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/meta.json create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/seconds.csv create mode 100644 evidence/track1/raw/Aprime/Ap4/rep3/steps.json create mode 100644 evidence/track1/raw/B/B1/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep1/meta.json create mode 100644 evidence/track1/raw/B/B1/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep1/seconds.csv create mode 100644 evidence/track1/raw/B/B1/rep1/steps.json create mode 100644 evidence/track1/raw/B/B1/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep2/meta.json create mode 100644 evidence/track1/raw/B/B1/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep2/seconds.csv create mode 100644 evidence/track1/raw/B/B1/rep2/steps.json create mode 100644 evidence/track1/raw/B/B1/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep3/meta.json create mode 100644 evidence/track1/raw/B/B1/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B1/rep3/seconds.csv create mode 100644 evidence/track1/raw/B/B1/rep3/steps.json create mode 100644 evidence/track1/raw/B/B3/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep1/meta.json create mode 100644 evidence/track1/raw/B/B3/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep1/seconds.csv create mode 100644 evidence/track1/raw/B/B3/rep1/steps.json create mode 100644 evidence/track1/raw/B/B3/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep2/meta.json create mode 100644 evidence/track1/raw/B/B3/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep2/seconds.csv create mode 100644 evidence/track1/raw/B/B3/rep2/steps.json create mode 100644 evidence/track1/raw/B/B3/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep3/meta.json create mode 100644 evidence/track1/raw/B/B3/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B3/rep3/seconds.csv create mode 100644 evidence/track1/raw/B/B3/rep3/steps.json create mode 100644 evidence/track1/raw/B/B4/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep1/meta.json create mode 100644 evidence/track1/raw/B/B4/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep1/seconds.csv create mode 100644 evidence/track1/raw/B/B4/rep1/steps.json create mode 100644 evidence/track1/raw/B/B4/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep2/meta.json create mode 100644 evidence/track1/raw/B/B4/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep2/seconds.csv create mode 100644 evidence/track1/raw/B/B4/rep2/steps.json create mode 100644 evidence/track1/raw/B/B4/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep3/meta.json create mode 100644 evidence/track1/raw/B/B4/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/B/B4/rep3/seconds.csv create mode 100644 evidence/track1/raw/B/B4/rep3/steps.json create mode 100644 evidence/track1/raw/C/R1m0/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep1/meta.json create mode 100644 evidence/track1/raw/C/R1m0/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R1m0/rep1/steps.json create mode 100644 evidence/track1/raw/C/R1m0/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep2/meta.json create mode 100644 evidence/track1/raw/C/R1m0/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R1m0/rep2/steps.json create mode 100644 evidence/track1/raw/C/R1m0/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep3/meta.json create mode 100644 evidence/track1/raw/C/R1m0/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m0/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R1m0/rep3/steps.json create mode 100644 evidence/track1/raw/C/R1m10/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep1/meta.json create mode 100644 evidence/track1/raw/C/R1m10/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R1m10/rep1/steps.json create mode 100644 evidence/track1/raw/C/R1m10/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep2/meta.json create mode 100644 evidence/track1/raw/C/R1m10/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R1m10/rep2/steps.json create mode 100644 evidence/track1/raw/C/R1m10/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep3/meta.json create mode 100644 evidence/track1/raw/C/R1m10/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m10/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R1m10/rep3/steps.json create mode 100644 evidence/track1/raw/C/R1m20/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep1/meta.json create mode 100644 evidence/track1/raw/C/R1m20/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R1m20/rep1/steps.json create mode 100644 evidence/track1/raw/C/R1m20/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep2/meta.json create mode 100644 evidence/track1/raw/C/R1m20/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R1m20/rep2/steps.json create mode 100644 evidence/track1/raw/C/R1m20/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep3/meta.json create mode 100644 evidence/track1/raw/C/R1m20/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m20/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R1m20/rep3/steps.json create mode 100644 evidence/track1/raw/C/R1m40/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep1/meta.json create mode 100644 evidence/track1/raw/C/R1m40/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R1m40/rep1/steps.json create mode 100644 evidence/track1/raw/C/R1m40/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep2/meta.json create mode 100644 evidence/track1/raw/C/R1m40/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R1m40/rep2/steps.json create mode 100644 evidence/track1/raw/C/R1m40/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep3/meta.json create mode 100644 evidence/track1/raw/C/R1m40/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R1m40/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R1m40/rep3/steps.json create mode 100644 evidence/track1/raw/C/R4m0/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep1/meta.json create mode 100644 evidence/track1/raw/C/R4m0/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R4m0/rep1/steps.json create mode 100644 evidence/track1/raw/C/R4m0/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep2/meta.json create mode 100644 evidence/track1/raw/C/R4m0/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R4m0/rep2/steps.json create mode 100644 evidence/track1/raw/C/R4m0/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep3/meta.json create mode 100644 evidence/track1/raw/C/R4m0/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m0/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R4m0/rep3/steps.json create mode 100644 evidence/track1/raw/C/R4m20/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep1/meta.json create mode 100644 evidence/track1/raw/C/R4m20/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep1/seconds.csv create mode 100644 evidence/track1/raw/C/R4m20/rep1/steps.json create mode 100644 evidence/track1/raw/C/R4m20/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep2/meta.json create mode 100644 evidence/track1/raw/C/R4m20/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep2/seconds.csv create mode 100644 evidence/track1/raw/C/R4m20/rep2/steps.json create mode 100644 evidence/track1/raw/C/R4m20/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep3/meta.json create mode 100644 evidence/track1/raw/C/R4m20/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/C/R4m20/rep3/seconds.csv create mode 100644 evidence/track1/raw/C/R4m20/rep3/steps.json create mode 100644 evidence/track1/raw/control/A2/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep1/meta.json create mode 100644 evidence/track1/raw/control/A2/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep1/seconds.csv create mode 100644 evidence/track1/raw/control/A2/rep1/steps.json create mode 100644 evidence/track1/raw/control/A2/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep2/meta.json create mode 100644 evidence/track1/raw/control/A2/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep2/seconds.csv create mode 100644 evidence/track1/raw/control/A2/rep2/steps.json create mode 100644 evidence/track1/raw/control/A2/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep3/meta.json create mode 100644 evidence/track1/raw/control/A2/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2/rep3/seconds.csv create mode 100644 evidence/track1/raw/control/A2/rep3/steps.json create mode 100644 evidence/track1/raw/control/A2pg2/rep1/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep1/meta.json create mode 100644 evidence/track1/raw/control/A2pg2/rep1/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep1/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep1/seconds.csv create mode 100644 evidence/track1/raw/control/A2pg2/rep1/steps.json create mode 100644 evidence/track1/raw/control/A2pg2/rep2/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep2/meta.json create mode 100644 evidence/track1/raw/control/A2pg2/rep2/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep2/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep2/seconds.csv create mode 100644 evidence/track1/raw/control/A2pg2/rep2/steps.json create mode 100644 evidence/track1/raw/control/A2pg2/rep3/docker-stats.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep3/meta.json create mode 100644 evidence/track1/raw/control/A2pg2/rep3/metrics.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep3/postgres.jsonl.gz create mode 100644 evidence/track1/raw/control/A2pg2/rep3/seconds.csv create mode 100644 evidence/track1/raw/control/A2pg2/rep3/steps.json create mode 100644 evidence/track1/summary.csv diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e6e63eb2..8a21e3ea 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 3 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бакулин, БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: трек 1 «Цена масштабирования» -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесено решение lab2 (PostgreSQL, Bearer-сессии в Valkey, Argon2id, keyset-пагинация). Добавлено: Compose + HAProxy 3.4 (реплики из DNS Docker, активные проверки `/health/ready`, повтор только при ошибке соединения, `maxconn 20` и `timeout queue 300ms` на реплику, свои 502/503/504 в формате контракта, `X-Request-Id` от балансировщика); `X-Instance-Id` — случайный alias процесса; SIGTERM-drain (readiness 503, 2 с обслуживания, затем shutdown); миграции в одноразовом сервисе `migrate`; общая идемпотентность отправки сообщения (таблица ключей, один SQL-оператор, 409 на другое содержание, чистка с запасом); `stop-one`, `instances`, rolling update через docker rollout; генератор нагрузки `tools/loadgen` и сценарии экспериментов `scripts/bench`. Исправления базы lab2: TTL-тесты ждут истечения с допуском на расхождение часов VM, тест повторных миграций считает файлы. Подробности — в REPORT.md. +Команды запуска из чистого clone: `make secrets && make up`, затем `make test`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` (на Windows `.cmd`), `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make instances`, `make rollout`, `make concurrency`, `make unit` (нужен Docker); эксперименты — `make failover-load`, `make rollout-load`, `make bench`, `make plot`; удаление данных — `make destroy`. +Результаты проверок и ссылка на REPORT.md: REPORT.md, разделы «Доказательства критериев» и «Эксперименты и отказы»; файлы в `evidence/` и `evidence/track1/`. +Ограничения / известные ошибки: один хост; HAProxy, PostgreSQL и Valkey — единственные экземпляры; точка насыщения — CPU PostgreSQL (~1400 запросов/с на 1 CPU); измерения на ноутбуке шумные, сравнения сделаны внутри одного захода; без TLS (lab7). Подробно — в REPORT.md. +Источники и использование ИИ: перечислены в REPORT.md. ИИ-ассистент использовался как парный программист: генерация кода, тестов и черновиков документации по моим архитектурным решениям с последующим ревью каждого модуля и переработкой по замечаниям; выбор стека, структура проекта, модель данных, обработка ошибок, логирование и границы модулей определены мной; все проверки воспроизведены мной вручную. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index c2679005..435a127a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,342 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бакулин, БПИ2301 +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Цена масштабирования». Уровни 3 и 4 выполнены полностью. +- Commit/PR: ветка `solution/bpi2301-bakulin`; ссылка на PR — после публикации. +- Использованные источники, библиотеки и помощь ИИ: HAProxy 3.4 Configuration Manual (`server-template`, `resolvers`, `http-check`, `retry-on`, `maxconn`, `timeout queue`, `http-error`), спецификация Docker Compose (`deploy.replicas`, `resources`, `healthcheck.start_interval`, `depends_on: service_completed_successfully`), [docker rollout](https://github.com/wowu/docker-rollout) v0.14 (MIT), документация PostgreSQL 18 (`INSERT … ON CONFLICT`, `FOR UPDATE SKIP LOCKED`, `pg_stat_database`, `pg_stat_activity`), документация Go (`net/http` `Server.Shutdown`, учёт CPU-лимита контейнера в `GOMAXPROCS`), документация Kubernetes о завершении пода и `preStop` и AWS ELB о deregistration delay (обоснование drain-задержки), Site Reliability Engineering, гл. 21–22 (Handling Overload, Addressing Cascading Failures), Bronson et al. «Metastable Failures in Distributed Systems» (HotOS 2021), G. Tene о coordinated omission (open-loop генератор). Библиотеки: `github.com/gin-gonic/gin` v1.12.0, `github.com/google/uuid` v1.6.0, `github.com/jackc/pgx/v5` v5.11.0 (драйвер и пул), `github.com/pressly/goose/v3` v3.27.0 (миграции), `github.com/redis/go-redis/v9` v9.23.0 (клиент Valkey), `golang.org/x/crypto/argon2`, в тестах `github.com/testcontainers/testcontainers-go` v0.39.0, для графиков matplotlib 3.10.3. ИИ-ассистент использовался как парный программист: генерация кода, тестов и черновиков документации по моим архитектурным решениям с последующим ревью каждого модуля и переработкой по замечаниям; выбор стека, структура проекта, модель данных, обработка ошибок, логирование и границы модулей определены мной; все проверки воспроизведены мной вручную. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Версии, на которых выполнялись проверки: Go 1.27.1 (`go.mod`), Docker Desktop 28.5.1 с Compose v2 (нужен 2.24+ из-за `env_file … required: false`), Python 3.13 для публичных проверок (нужен 3.9+), GNU Make. Образы закреплены digest'ами: `haproxy:3.4.7-alpine3.24`, `postgres:18.6-alpine3.24`, `valkey/valkey:9.1.2-alpine3.24` в `compose.yaml`, `golang:1.27.1-alpine3.23` и `alpine:3.23` в `Dockerfile`. + +```sh +make secrets # ./secrets: пароли PostgreSQL и Valkey и URL для API; существующие файлы не перезаписываются +make up # docker compose up -d --build --wait: postgres, valkey → migrate (один раз) → 2 реплики api; lb готов, когда через него отвечает реплика +make check PYTHON=python +make test PYTHON=python +make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python # на Windows: scripts/stop-one.cmd +docker compose start api # вернуть остановленную реплику +make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python # на Windows: scripts/restart.cmd +make instances # какой контейнер отвечает под каким X-Instance-Id +make rollout # rolling update без простоя (docker rollout, скачивается с проверкой SHA-256) +make concurrency # гонки отправки и общий Idempotency-Key через разные реплики +make unit # собственные тесты; нужен Docker: тесты поднимают свои PostgreSQL и Valkey +``` + +Число реплик: `API_REPLICAS=3 make up`. Если порт 8080 занят — `PUBLISH=127.0.0.1:28080 make up` и `BASE_URL=http://localhost:28080` для проверок; страница статистики HAProxy — `http://127.0.0.1:8404/` (`STATS_PUBLISH`). + +Эксперименты (настройки и генератор нагрузки — в `compose.bench.yaml`, скрипты подключают его сами): + +```sh +make failover-load KIND=stop REPLICA=1 # отказ реплики под нагрузкой: stop (SIGTERM) или kill (SIGKILL) +make rollout-load # rolling update под нагрузкой +make bench SERIES="scripts/bench/series/A.conf …" RATES=400,800,… # трек 1; каждый прогон удаляет данные стенда +make plot PYTHON= +``` + +Остановка и удаление данных. `docker compose stop` — graceful остановка всего стенда; `make down` — удалить контейнеры с сохранением volumes; `make destroy` (`docker compose down -v`) — **намеренное удаление всех учебных данных**. `make bench` тоже начинает каждый прогон с пустых volumes. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` + edge data (internal: без выхода наружу) +клиент ──> lb (HAProxy 3.4) ──roundrobin──> api × N (Go) ──pgxpool ≤10──> postgres [pgdata] источник истины + 127.0.0.1:8080 │ проверки /health/ready │ └──go-redis──────> valkey [valkeydata] сессии + 127.0.0.1:8404 │ DNS Docker: server-template │ + (статистика) └─ свои 502/503/504 в формате migrate (одноразовый, goose) ──> postgres + контракта loadgen (только эксперименты, compose.bench.yaml) +``` + +Реплики взаимозаменяемы: сессии лежат в Valkey, всё остальное — в PostgreSQL; ни одна реплика не хранит состояния, от которого зависит корректность (README:18). Порты публикует только `lb`, и только на loopback; у `api` нет `ports` и `container_name`, число реплик — `deploy.replicas: ${API_REPLICAS:-2}` (README:37). Сеть `data` объявлена `internal`: хранилища недоступны ни снаружи, ни из `lb`. + +### Балансировщик: Compose + HAProxy + +Выбор и альтернативы: +- **Nginx OSS** проверяет backend'ы только пассивно (`max_fails`/`fail_timeout`): упавшую реплику он замечает по неудачному клиентскому запросу, периода проверки нет, а README:35 требует назвать период и время обнаружения. Главное — он не видит readiness: реплика, которая по SIGTERM уже отвечает `/health/ready` 503, продолжает получать запросы до закрытия порта. Активный `health_check` есть только в NGINX Plus. +- **Traefik** с docker-провайдером требует смонтированный `docker.sock`, то есть фактически root на хосте — против README:37 («общие только там, где нужно»). +- **Docker Swarm** даёт rolling update и самовосстановление из коробки, но `stack deploy` не собирает образ, не читает `.env` и игнорирует `depends_on` (порядок migrate → api пришлось бы писать в коде), а встроенная балансировка (IPVS) делит TCP-соединения, не HTTP-запросы, без повторов и без HTTP-проверок. Для одного хоста Compose продолжает lab2 и запускается одной командой. + +HAProxy (`infra/haproxy/haproxy.cfg`): +- **Обнаружение реплик** — `server-template api 10 api:8080` + `resolvers docker` (DNS Docker, опрос раз в 1 с, `hold obsolete 0s`): новые, перезапущенные и заменённые реплики появляются без перезагрузки LB, исчезнувший адрес уходит сразу. `init-addr none` — LB стартует раньше реплик. Десять слотов — до пяти реплик, удвоенных на время rolling update. +- **Активная проверка** `GET /health/ready`: `inter 1s`, после первой неудачи `fastinter 500ms`, `fall 2` / `rise 2`, `timeout check 2s`. Расчётное время обнаружения не готовой реплики — до 1,5 с (+ до 2 с, если проверка зависла до таймаута). +- **Балансировка** `roundrobin`, без sticky-сессий (README:19). Запросы разные по весу (вход с Argon2id тяжелее GET); при перекосе `balance leastconn` меняется одним словом — в экспериментах перекоса между репликами не видно (раскладка по alias в `seconds.csv`). +- **Повторы** только при ошибке соединения: `retry-on conn-failure`, `retries 2`, `option redispatch 1` — повтор уходит на другую реплику. Это безопасно для любого `POST`: запрос не дошёл до API. Таймаут или пустой ответ не повторяются — запись могла уже произойти (contracts/README:43). +- **Ограничение на реплику** (улучшение C трека 1): `maxconn 20` на сервер и `timeout queue 300ms`. Сверх 20 одновременных запросов на реплику запросы ждут в очереди LB не дольше 300 мс, затем получают 503. 20 = 10 соединений пула реплики + запас на запросы без БД; эта связь настроек LB и пула — осознанная. +- **Свои ошибки LB** (502/503/504) — JSON в формате контракта с `request_id`; `X-Instance-Id` у них нет, так как ни одна реплика не отвечала (contracts/README:39). +- **X-Request-Id**: LB генерирует id (32 hex) и ставит его в запрос (`http-request set-header`, перезаписывая значение клиента); API сохраняет входящий id, если он соответствует `^[A-Za-z0-9-]{1,64}$`, иначе генерирует свой. Один id проходит через лог LB, лог реплики, заголовок и тело ошибки; повтор на другую реплику идёт под тем же id. Клиент подставить свой id не может: API доступен только через LB. +- **Лог** HAProxy пишет реплику, alias и request id каждого запроса — по ним находится строка в логе реплики. Страница статистики на `127.0.0.1:8404` показывает состояние и последнюю проверку каждой реплики. + +### X-Instance-Id + +Случайный alias процесса — 8 байт `crypto/rand` в hex, задаётся при старте и не меняется до выхода (contracts/README:39, задел на lab7: hostname/IP не раскрываются). Ставится первым middleware, поэтому есть на каждом ответе API, включая ошибки и health. Связь alias ↔ контейнер показывает `make instances` (`scripts/api-instances.sh` спрашивает каждый работающий контейнер через `docker exec … wget /health/live`). Перезапуск процесса даёт новый alias. + +### SIGTERM, readiness и таймауты + +| Параметр | Значение | Где | +| --- | --- | --- | +| Проверка LB | GET /health/ready каждую 1 с (0,5 с после неудачи), DOWN после 2 неудач, UP после 2 удач, таймаут 2 с | haproxy.cfg | +| /health/ready | 503 при shutdown или если PostgreSQL/Valkey не отвечают за общий бюджет 1 с | internal/health | +| По SIGTERM | readiness → 503, процесс продолжает обслуживать 2 с (`APP_DRAIN_DELAY`), затем `http.Server.Shutdown` с таймаутом 8 с | internal/server | +| Docker | `stop_grace_period: 15s` > 2 + 8 с | compose.yaml | +| Дедлайн запроса API | 5 с; `WriteTimeout` 10 с, чтобы 503 успел записаться | internal/app, internal/server | +| LB → API | `timeout connect 1s`, `timeout server 7s` (> 5 с: медленный запрос заканчивается 503 самого API с причиной в логе; 504 LB — только для зависшей реплики) | haproxy.cfg | +| PostgreSQL | пул 10 на реплику, `statement_timeout` 5 с, `idle_in_transaction_session_timeout` 10 с, connect и DNS 1 с | internal/platform/postgres | + +Drain-задержка (2 с) больше расчётного времени обнаружения LB (1,5 с), поэтому LB выводит реплику из ротации раньше, чем она закрывает порт. Измерено (evidence/failover/stop-*): от SIGTERM до DOWN в HAProxy — 0,67 и 1,38 с. При SIGKILL реплика исчезает из DNS Docker и LB переводит слот в MAINT по резолверу — 0,67 и 0,69 с после выхода процесса; запросы, попавшие на мёртвый адрес в этом окне, LB повторяет на другой реплике (ошибка соединения). + +Так же устроено в продакшене: в Kubernetes под при удалении сначала исключается из Endpoints, а `preStop`-задержка держит процесс живым, пока это доходит до балансировщиков; AWS ELB/ALB использует deregistration delay. Общий принцип — задержка дренажа больше времени, за которое балансировщик узнаёт об остановке. + +### Миграции + +Схема применяется одноразовым сервисом `migrate` (`cmd/migrate`, тот же образ со своим entrypoint): он ждёт здоровый PostgreSQL, применяет миграции goose под session advisory lock и завершается; `api` стартует только после `service_completed_successfully` (README:18 — отдельно, не гонкой реплик). API схему больше не меняет. Правило совместимости для rolling update (README:38): новые миграции выполняются, пока старые реплики ещё работают, поэтому миграция релиза только добавляет; удаление и переименование переносятся в следующий релиз. Миграция lab3 — новая таблица, старый код её не видит. + +### Общая идемпотентность отправки + +Таблица `message_idempotency_keys` (`migrations/00002_message_idempotency_keys.sql`): PK `(user_id, chat_id, key)`, FK `(chat_id, message_id) → messages`, индекс по `created_at`. Область ключа по контракту — пользователь + метод + путь чата + ключ (contracts/README:40); ключ принимает только `POST /chats/{chat_id}/messages`, поэтому метод и путь однозначно заданы чатом. Ключи вынесены в отдельную таблицу: они не атрибут сообщения, а инфраструктура вокруг него. + +Порядок проверок `POST /chats/{chat_id}/messages`: сессия (middleware, 401) → `chat_id` (400) → заголовок `Idempotency-Key`: ровно одно значение `[A-Za-z0-9._:-]{1,128}` (400) → JSON (400) → членство (404, как у несуществующего чата) → текст (400) → запись. Повтор проходит те же проверки заново: отозванная сессия или потерянный доступ не получают сохранённый результат (contracts/README:42). + +Запись — один SQL-оператор (CTE): вставка ключа `ON CONFLICT … DO UPDATE … WHERE created_at < now() - TTL` и вставка сообщения из `RETURNING`. Ключ и сообщение пишутся атомарно (contracts/README:42), блокировка ключа не переживает сетевого обмена с репликой. Конкурентный второй запрос с тем же ключом ждёт коммита первого на уникальном индексе и получает пустой `claimed`. Тогда вторым оператором читается сохранённое сообщение (снимок первого оператора не видит только что закоммиченный ключ). Тот же текст — 201 и то же сообщение; другой текст — 409 (contracts/README:41); сравнение содержания — `Message.SameContent`. Ключ старше 24 ч (`IDEMPOTENCY_TTL_SECONDS`) перехватывается новым сообщением. + +Чистка: каждая реплика раз в 10 мин удаляет ключи старше TTL + 1 мин пачками по 1000 (`DELETE … WHERE (…) IN (SELECT … FOR UPDATE SKIP LOCKED LIMIT 1000)`): чистильщики разных реплик и запросы, держащие ключ, не ждут друг друга. Запас в минуту гарантирует, что ключ, который запрос только что увидел действующим, не исчезнет до чтения сохранённого сообщения (оба `now()` — часы одного PostgreSQL, между операторами доли секунды при дедлайне 5 с). + +Проверено: 20 параллельных запросов с одним ключом — одно сообщение (`TestConcurrentAppendOnceSameKeyStoresOneMessage`); два приложения над общими хранилищами — одно сообщение (`TestConcurrentReplaysAcrossTwoAppsCreateOneMessage`); через LB 20 раундов × 8 параллельных попыток на соединениях — 20 сообщений, ответы двух разных реплик (evidence/concurrency.json). + +### Rolling update + +`make rollout` (`scripts/rollout.sh`) использует готовый [docker rollout](https://github.com/wowu/docker-rollout) v0.14 (MIT; скачивается по тегу и проверяется SHA-256): рядом со старыми репликами поднимается столько же новых, после того как все новые healthy (и ещё 4 с, чтобы LB их увидел), старые останавливаются через тот же SIGTERM-drain; если новая реплика не стала healthy, удаляются новые, старые продолжают работать — смеси версий не остаётся. Перед этим скрипт пересоздаёт `migrate` из нового образа без запуска: `compose up` внутри docker rollout применяет новые миграции один раз до старта новых реплик. Если какая-то реплика остановлена (например, после `stop-one`), скрипт отказывается работать: docker rollout заменил бы только её и оставил две версии. + +### Лимиты и конфигурация + +`compose.yaml` — стенд с фиксированными лимитами у каждого сервиса: lb 0,5 CPU / 128M, api 1,0 CPU / 256M на реплику (`GOMEMLIMIT=230MiB`: Go не читает лимит cgroup), migrate 0,5 / 128M, postgres 1,0 / 512M (`max_connections=150` = 10 слотов LB × пул 10 + migrate и psql), valkey 0,5 / 320M. Всё, что меняют эксперименты (лимиты API и PostgreSQL, пул, `maxconn`, лог LB, генератор нагрузки), — в `compose.bench.yaml`, который подключают только `scripts/bench/*`; без переменных он не меняет стенд. + +Соответствие `.env.example`: + +| Переменная | `.env.example` | Реализация | По умолчанию | +| --- | --- | --- | --- | +| APP_HOST, APP_PORT, APP_ENV | да | как в lab2 | 0.0.0.0, 8080, lab | +| SESSION_TTL_SECONDS | да | срок сессии | 3600 | +| DATABASE_URL_FILE | да | файл с DSN, обязателен | `/run/secrets/database_url` | +| SESSION_STORE, CACHE_URL_FILE | да | не используются: сессии всегда в Valkey, кэша нет | — | +| API_REPLICAS | да | `deploy.replicas` сервиса api | 2 | +| IDEMPOTENCY_TTL_SECONDS | да | срок ключа сообщения | 86400 | +| SESSION_STORE_URL_FILE | нет | файл с URL Valkey, обязателен | `/run/secrets/session_store_url` | +| APP_DRAIN_DELAY | нет | обслуживание после SIGTERM с readiness 503 | 2s | +| DB_MAX_CONNS | нет | пул PostgreSQL на реплику (переменная трека 1) | 10 | +| APP_DEBUG_ADDR | нет | внутренний expvar-listener для экспериментов, наружу не публикуется | выключен | +| APP_LOG_LEVEL, APP_SHUTDOWN_TIMEOUT, APP_MAX_BODY_BYTES | нет | как в lab2 | info, 8s, 65536 | + +Роли стенда: + +| Роль | Сервис | Версия | Health/readiness | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| load_balancer | lb | haproxy 3.4.7-alpine3.24 @digest | wget `/health/ready` через себя | — | edge; `127.0.0.1:8080`, статистика `127.0.0.1:8404` | — | +| api | api × N | сборка из исходников | `/health/ready` (PG + Valkey) | — | edge + data, без портов | secrets `database_url`, `session_store_url` | +| migrations | migrate | тот же образ | одноразовый | — | data | secret `database_url` | +| postgres | postgres | 18.6-alpine3.24 @digest | `pg_isready -h 127.0.0.1` | `pgdata` | data | secret `postgres_password` | +| redis_or_valkey | valkey | 9.1.2-alpine3.24 @digest | `valkey-cli ping` с паролем | `valkeydata` (AOF) | data | secret `valkey_auth` | + +### Что перенесено из lab2 и что изменено + +Перенесено без изменений: модель данных и keyset-пагинация, Bearer-сессии в Valkey, Argon2id, формат ошибок, таймауты зависимостей. Изменено в lab3: +- `X-Instance-Id` на каждом ответе; `X-Request-Id` принимается от LB; +- обязательный `Idempotency-Key` и таблица ключей, `Send(ctx, actor, MessagePayload)`; +- graceful shutdown с drain-задержкой; +- миграции вынесены из API в `cmd/migrate`; +- `DB_MAX_CONNS`, внутренний expvar-listener (`APP_DEBUG_ADDR`) для экспериментов; +- `tools/concurrency` — вторая фаза: общий ключ через разные реплики; +- новые `tools/loadgen` и `scripts/bench/*`. + +Исправления базы lab2: оба `TestSessionExpiresAfterTTL` (`internal/app`, `internal/auth`) ждут истечения с допуском — Valkey в VM Docker, его часы отстают от Windows на доли секунды; `TestMigrateIsRepeatable` считает файлы миграций, а не ждёт ровно одну. + +CI (`.github/workflows/app.yml`): unit с race detector, затем стенд, smoke, persistence, failover со `stop-one.sh`, обе фазы concurrency, логи и `down -v`. ## Доказательства критериев +Файлы уровней 3 и 4 сняты на одном стенде, поднятом с пустых volumes; в заголовке каждого файла — дата, commit и параметры хоста (Windows 11, Docker Desktop 28.5.1, WSL2-VM 8 vCPU / 12 GB, i5-1240P). Запуски под нагрузкой (`evidence/failover`, `evidence/rollout`) записывают то же в `meta.json`; все их времена — по часам VM Docker (`docker inspect`, `compose logs --timestamps`, генератор), часы Windows расходятся с ними на 0,3–0,6 с. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make check`, `make test` | оба PASS (профиль lab3, включая идемпотентность) | evidence/smoke.txt | +| Стенд в compose.yaml, одна команда запуска; ≥2 API, одна точка входа, общие БД и сессии (README:25) | `docker compose config --quiet`, `docker compose up -d --build --wait` с пустых volumes | migrate применил 2 миграции и завершился, 2 реплики api healthy; порты только у lb; `\dt`: 6 таблиц, включая `message_idempotency_keys` | evidence/bootstrap.txt | +| Разные запросы одного пользователя — разные реплики без повторного входа (README:26) | 10 GET с одним токеном через LB; `make instances` | alias чередуются `c6f9…`/`0d2f…` (round-robin), оба alias принадлежат разным работающим контейнерам | evidence/bootstrap.txt | +| Остановка любой одной реплики: чтение и запись не позже 10 с, подтверждённое доступно (README:27) | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` дважды | PASS оба раза: 127 и 126 успешных записей, 0 неудачных попыток, первое восстановление через 4,6 и 5,2 с (метрика теста); первый раз остановлен `lab3-api-2`, второй — `lab3-api-1` (`instances` до и после) | evidence/failover.txt | +| Повтор POST с тем же ключом не создаёт дубль, в том числе конкурентный и через другую реплику (README:28) | `make concurrency` (фаза 2); `TestConcurrentAppendOnceSameKeyStoresOneMessage`, `TestConcurrentReplaysAcrossTwoAppsCreateOneMessage`, `TestReplayWithDifferentBodyIs409`, `TestReplayRechecksSession` | 20 раундов × 8 параллельных попыток с одним ключом — 20 сообщений, ответы двух реплик, другой текст — 409 | evidence/concurrency.json, evidence/concurrency.txt, evidence/tests.txt | +| make test, persistence и failover проходят (README:29) | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS тем же токеном; `StartedAt` всех пяти контейнеров изменился | evidence/persistence.txt | +| Гонки отправки | `make concurrency` (фаза 1) | 10 × 10 — 100 уникальных сообщений в порядке id | evidence/concurrency.json | +| Health/readiness, timeouts, SIGTERM согласованы; LB исключает backend; период и время обнаружения (README:35) | конфигурация (раздел выше), `TestDrainKeepsServingWhileNotReady`; `make failover-load KIND=stop` для каждой реплики | период 1 с (0,5 с после неудачи), DOWN после 2 неудач; от SIGTERM до DOWN 1,43 и 0,96 с, порт закрыт через 2,0 с после SIGTERM; 0 ошибок | evidence/failover/stop-1, stop-2, evidence/tests.txt | +| Каждая реплика по очереди: RPS, ошибки, время восстановления (README:36) | `scripts/bench/failover.sh stop|kill 1|2` (50 записей/с, 60 с, отказ на 15-й с, возврат на 35-й) | 4 прогона × 3000 сообщений: 3000 подтверждено, 0 ошибок, 0 повторов, 0 дублей, 0 потерь; детали в таблице ниже | evidence/failover/* | +| Лимиты CPU/RAM, API без портов в обход LB, нет container_name (README:37) | `compose.yaml`, `docker compose ps` | лимиты у каждого сервиса, `ports` только у lb | compose.yaml, evidence/bootstrap.txt | +| Rolling update под нагрузкой, миграции один раз (README:38) | `make rollout-load` (50 записей/с, обновление на 15-й с) | обе реплики заменены (exit 0), 3000/3000 подтверждено, 0 ошибок, ответы 4 разных процессов; `migrations applied` — один раз | evidence/rollout | +| Выбранный трек 1 «Цена масштабирования» | серии A, A′, B, парный контроль и C (`make bench`, `make plot`), 3 повтора вперемешку | 1/2/4 реплики при одном бюджете — одна ёмкость, +33 % CPU API на запрос; точка насыщения — CPU PostgreSQL ~1400 запросов/с; удвоение CPU базы — 1200 → 2000; `maxconn 20` — за насыщением 1013–1594 успешных/с вместо 93–484 | evidence/track1 | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Уровень 4: отказ реплики и rolling update под нагрузкой + +Постоянная запись 50 сообщений/с одним пользователем в течение 60 с (`tools/loadgen timeline`); каждый неподтверждённый `POST` генератор повторяет с тем же ключом; в конце вся история читается обратно — каждое подтверждённое сообщение должно быть ровно один раз. Стенд — `compose.yaml` (2 реплики по 1 CPU), генератор работает в сети Docker и обращается к `lb:8080`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker stop` реплики 1 (SIGTERM) на 15-й с | readiness 503, LB выводит реплику до закрытия порта, ошибок нет | DOWN через 1,43 с после SIGTERM, порт закрыт через 2,0 с; 3000/3000 подтверждено, 0 ошибок, 0 повторов | после `start` UP через 0,57 с; 0 дублей, 0 потерь | +| `docker stop` реплики 2 | то же | DOWN через 0,96 с; 3000/3000, 0 ошибок | UP через 1,03 с; 0 дублей, 0 потерь | +| `docker kill` реплики 1 (SIGKILL) | обрыв соединений; не дошедшее до API LB повторяет на другой реплике; неподтверждённое генератор повторяет с ключом | адрес исчез из DNS, слот в MAINT через 0,77 с после выхода процесса; 3000/3000, 0 ошибок, 0 повторов генератора | UP через 0,47 с после старта; 0 дублей, 0 потерь | +| `docker kill` реплики 2 | то же | MAINT через 0,91 с; 3000/3000, 0 ошибок | UP через 0,68 с; 0 дублей, 0 потерь | +| `make rollout` под нагрузкой на 15-й с | замена обеих реплик без ошибок, миграции один раз | exit 0 за ~16 с, 3000/3000, 0 ошибок, ответы 4 процессов (2 старых, 2 новых); `migrations applied` один раз | 0 дублей, 0 потерь | + +При 50 записях/с и ответах за единицы миллисекунд в момент SIGKILL в полёте почти никогда нет запроса, поэтому неподтверждённых `POST` не было. Такой запрос получил бы 502 от LB (повторять его LB не вправе), и генератор повторил бы его с тем же ключом; этот путь проверяют `make test-failover` и фаза 2 `make concurrency`. После перезапуска контейнера HAProxy возвращает слот в UP при повторном разрешении DNS, не дожидаясь `rise 2`; если процесс ещё не открыл порт, запрос получает отказ соединения и повторяется на другой реплике (`retry-on conn-failure`). + +### Трек 1. Цена масштабирования + +#### Гипотезы (сформулированы до измерений) + +- **H1. Фиксированный бюджет.** При одном и том же суммарном CPU/RAM API (2 CPU, 1 GiB) переход 1 → 2 → 4 реплики не увеличивает ёмкость (в пределах ±10 %), а хвост задержек ухудшает: у каждой реплики свои накладные расходы (GC, пул, рантайм), а дробный лимит 0,5 CPU троттлится квотой CFS внутри периода 100 мс. Реплики дают отказоустойчивость, а не производительность. +- **H2. Рост ресурсов.** Реплики фиксированного размера (1 CPU) увеличивают ёмкость почти линейно, пока она не упрётся в общий ресурс; ожидаемо — PostgreSQL с 1 CPU: его CPU у лимита, а ёмкость стоит, сколько реплик ни добавляй. +- **H3. Скрытая переменная — пул.** Пул задан на реплику (10), поэтому 4 реплики держат 40 соединений к БД против 10 у одной. Если пул одной реплики был узким местом, часть «выигрыша от реплик» — выигрыш от большего пула. Проверяется серией с фиксированным суммарным пулом. +- **Контроль.** Если узкое место — CPU PostgreSQL, удвоение его CPU поднимает ёмкость. Это проверка вмешательством, а не по корреляции «CPU у лимита». +- **H4 (улучшение C).** Без ограничения числа запросов на реплику перегрузка обрушивает успешную пропускную способность каскадом; с ограничением на LB (`maxconn`) успешная пропускная способность за точкой насыщения держится около ёмкости, а лишняя нагрузка быстро получает 503. + +Гипотеза H4 появилась из предварительных прогонов 2026-11-03 (вне evidence): за точкой насыщения успешных ответов становилось 7–300 в секунду вместо ~1300. Механизм виден в метриках: без ограничения HAProxy отдаёт в реплику тысячи одновременных запросов, они ждут пул внутри приложения; `/health/ready` ждёт в той же очереди и не укладывается в 2 с, LB выводит живую реплику, её нагрузка уходит на остальные, и они выпадают следом; запросы, брошенные клиентом по таймауту, база всё равно выполняет, а отменённые запросы рвут соединения пула (`total_conns` 10 → 1, +30–40 сессий PostgreSQL в секунду). + +#### Методика + +Всё, что ниже, выполняют скрипты из repo: `tools/loadgen` (генератор), `scripts/bench/run-series.sh` (серии), `scripts/bench/plot.py` (таблицы и графики), конфигурации — `scripts/bench/series/*.conf`, настройки — `compose.bench.yaml`. + +- **Стенд.** Windows 11, i5-1240P (4 P- + 8 E-ядер, 16 потоков), ноутбук от сети, схема «Высокая производительность»; WSL2-VM 8 vCPU / 12 GB; Docker Desktop 28.5.1. Во время прогонов тяжёлой посторонней нагрузки нет (допускался браузер). +- **Постоянные компоненты** во всех сериях: HAProxy 1 CPU / 128M (`nbthread 1`), PostgreSQL 1 CPU / 1 GiB (кроме контрольного прогона), Valkey 0,5 CPU / 320M, генератор 2 CPU / 1 GiB (`GOMEMLIMIT=900MiB`). Меняются только API (число реплик, CPU/RAM, `GOMEMLIMIT`, пул), `maxconn` и CPU PostgreSQL — каждая строка серии задаёт их явно. +- **Профиль.** 80 % — `GET /chats/{id}/messages?limit=50` (первая страница — 50 самых старых сообщений чата, размер ответа постоянен), 20 % — `POST` сообщения со свежим ключом; пользователь и чат случайны (seed фиксирован). Повторов нет: каждая ошибка — ошибка. +- **Данные.** Каждый прогон — свежий стенд с пустых volumes. Перед измерением через API: 200 пользователей, 100 чатов по 2–5 участников, по 50 сообщений в чате (5000 сообщений); вход каждого пользователя один раз — Argon2id в измерения не попадает. Прирост за прогон — `messages_written_during_steps` в `steps.json`. +- **Генератор open-loop.** Запрос i отправляется в момент start + i/rate, не дожидаясь ответов: медленная система не снижает себе нагрузку (нет coordinated omission). Таймаут клиента 2 с — класс `client_timeout`, это граница генератора, а не таймаут системы. Ошибки LB (нет `X-Instance-Id`) считаются отдельно от ошибок API. Генератор работает в сети Docker и обращается к `lb:8080`: проброс порта Windows → WSL2 в путь не попадает. +- **Ступени.** Прогрев 30 с на первой частоте, затем ступени по 30 с: 400, 800, 1000, 1200, 1400, 1600, 1800, 2000, 2400, 2800 запросов/с (C: 400–2400 через 400). В сериях A–B лестница останавливается после первой ступени, где система вышла из целевых границ (ошибок ≥ 1 % или p99 > 500 мс); контроль и C идут до конца (`STOP_ON_FAIL=0`). +- **Ёмкость** — наибольшая ступень, где ошибок < 1 %, p99 успешных ≤ 500 мс и ступень действительна: генератор отправлял по расписанию (p99 отставания ≤ 10 мс). Недействительная ступень означает, что ограничивал генератор, а не система; она в ёмкость не идёт и считается в `capacity.csv` отдельно. +- **Повторы.** 3 повтора каждой конфигурации, чередуются: сначала повтор 1 всех конфигураций, потом 2, потом 3, — медленный период хоста не ложится на одну конфигурацию. Итог — медиана повторов, на графике ёмкости — разброс. +- **Контроль шума.** Перед каждым прогоном — замер скорости хоста (фиксированный цикл в контейнере с 1 CPU, `host_probe_s` в `meta.json`) и CPU генератора на 1000 запросов на первой ступени; прогон с отклонением > 25 % от медианы помечается в `control.csv`. +- **Что снимается** (`evidence/track1/raw/<серия>/<конфигурация>/rep/`): `steps.json` (по ступеням: частоты, ошибки по классам, p50/p95/p99, отставание, alias), `seconds.csv` (посекундно), `metrics.jsonl.gz` (раз в секунду: HAProxy по репликам — сессии, очередь, состояние; expvar каждой реплики — пул pgx: ожидания, отменённые и новые соединения, GOMAXPROCS, память и GC), `docker-stats.jsonl.gz` (CPU/RAM контейнеров), `postgres.jsonl.gz` (коммиты, сессии, ожидания активных backend'ов), `compose.log` (переходы состояния LB, старт и остановка реплик), `meta.json` (commit, хост, параметры). + +Повторить: + +```sh +make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/Aprime.conf scripts/bench/series/B.conf" RATES=400,800,1000,1200,1400,1600,1800,2000,2400,2800 +STOP_ON_FAIL=0 make bench SERIES=scripts/bench/series/control.conf RATES=400,800,1000,1200,1400,1600,1800,2000,2400,2800 +STOP_ON_FAIL=0 make bench SERIES=scripts/bench/series/C.conf RATES=400,800,1200,1600,2000,2400 +make plot PYTHON= +``` + +#### Серии + +| Серия | Конфигурации | Вопрос | +| --- | --- | --- | +| A (H1) | A1 1×2,0 CPU / 1 GiB, A2 2×1,0 / 512M, A4 4×0,5 / 256M; пул 10 на реплику | меняется ли ёмкость, если тот же бюджет резать на реплики | +| A′ (H3) | A′1 1×2,0 с пулом 20, A′4 4×0,5 с пулом 5; A′2 = A2 | сколько разницы внутри A даёт пул | +| B (H2) | B1, B3, B4 — реплики по 1 CPU / 512M, пул 10; B2 = A2 | где ёмкость перестаёт расти и что у лимита | +| Контроль | A2 и A2 с PostgreSQL 2 CPU поочерёдно | поднимает ли CPU базы потолок | +| C (H4) | 1 и 4 реплики по 1 CPU: `maxconn` 0 (без ограничения) и 20; на 1 реплике ещё 10 и 40; лестница 400–2400 без остановки | что даёт ограничение за точкой насыщения и насколько важно значение 20 | + +Лимиты в B4 и C на 4 репликах: API 4 + LB 1 + PostgreSQL 1 + Valkey 0,5 + генератор 2 = 8,5 CPU лимитов на 8 vCPU; лимиты — потолки, фактически контейнеры занимают меньше (см. CPU в `summary.csv`), но на таких ступенях конкуренция за vCPU возможна и видна по отставанию генератора. + +#### Результаты + +Сырые данные — `evidence/track1/raw/`, сводки и графики — `evidence/track1/` (`summary.csv` по ступеням, `capacity.csv`, `control.csv`, `cost.csv`, PNG). Данные: 5000 сообщений перед измерением, за прогон пишется ещё до ~20 000. + +**Шум стенда.** Замер скорости хоста (один и тот же цикл) менялся от 2,55 до 4,86 с при медиане 2,93 с; 9 из 24 прогонов серий A–B отклонялись от медианы больше чем на 25 % по замеру или по CPU генератора (`control.csv`). Шум проявляется как короткие остановки всей VM: p99 на одной ступени подскакивает выше 500 мс при CPU компонентов далеко от лимита, затем снова падает. Поэтому ёмкость отдельного прогона по строгому правилу сильно шумит, и выводы строятся на трёх опорах: медиана ёмкости по повторам, CPU компонентов на запрос при одинаковой нагрузке (устойчиво к остановкам, считается только по непомеченным прогонам) и CPU компонентов на ступени 1400 запросов/с. + +**Ёмкость** (наибольшая прошедшая ступень; по повторам и медиана): + +| Конфигурация | API | Повторы | Медиана | +| --- | --- | --- | --- | +| A1 | 1 × 2 CPU, пул 10 | 1000, 1400, 1000 | 1000 | +| A2 = B2 = A′2 | 2 × 1 CPU, пул 10 | 1400, 1200, 1000 | 1200 | +| A4 | 4 × 0,5 CPU, пул 10 | 800, 1400, 1000 | 1000 | +| A′1 | 1 × 2 CPU, пул 20 | 1200, 1000, 400* | 1000 | +| A′4 | 4 × 0,5 CPU, пул 5 | 1400, 1000, 400* | 1000 | +| B1 | 1 × 1 CPU | 1000, 400**, 1200 | 1000 | +| B3 | 3 × 1 CPU | 1200, 400*, 1400 | 1200 | +| B4 | 4 × 1 CPU | 1400, 800*, 1400 | 1400 | + +\* прогон помечен контролем шума: ступень провалена при отставании генератора 11–23 мс. \*\* не шум, а метастабильный отказ (ниже). + +**CPU на запрос при 800–1000 запросах/с** (мс CPU, медиана непомеченных прогонов; `cost.csv`): + +| Конфигурация | API | PostgreSQL | HAProxy | Оценка потолка 1 CPU PostgreSQL | +| --- | --- | --- | --- | --- | +| A1 (1 × 2) | 1,15 | 0,74 | 0,33 | ~1340 запросов/с | +| A2 (2 × 1) | 1,40 | 0,74 | 0,34 | ~1350 | +| A4 (4 × 0,5) | 1,53 | 0,75 | 0,34 | ~1330 | +| A′1 (1 × 2, пул 20) | 0,98 | 0,58 | 0,27 | ~1710 | +| A′4 (4 × 0,5, пул 5) | 1,52 | 0,72 | 0,32 | ~1390 | +| B1 (1 × 1) | 1,01 | 0,60 | 0,30 | ~1670 | +| B3 (3 × 1) | 1,60 | 0,76 | 0,36 | ~1320 | +| B4 (4 × 1) | 1,55 | 0,71 | 0,34 | ~1410 | + +Оценка потолка — 1 CPU, делённый на цену запроса; при большей нагрузке цена немного падает, поэтому реальное насыщение — на 1400–1600. На ступени 1400 запросов/с CPU PostgreSQL — 90–102 % его лимита во всех конфигурациях, где ступень действительна (A1 91, A2 92, A4 97, A′1 90, A′4 97, B1 91, B3 102, B4 98–99 %), а ожидания соединения пула растут до 800–2000 в секунду. + +**H1 — подтверждена.** При одном бюджете 2 CPU ёмкость 1, 2 и 4 реплик одинакова в пределах шума (медианы 1000–1200, отдельные прогоны 800–1400). Цена разбиения видна по CPU: одна реплика тратит 1,15 мс CPU на запрос, две — 1,40, четыре — 1,53 (+33 %): у каждой реплики свой рантайм, GC и пул, а при лимите 0,5 CPU `GOMAXPROCS=2` и квота CFS троттлят процесс. В ёмкости это пока не видно, потому что раньше упирается база; но на 1400 запросах/с API A4 занят на 195 % из 200 %, а A1 — на 130 % из 200 %: при четырёх репликах API подходит к пределу одновременно с базой, и при ещё более мелких репликах ограничителем стал бы уже он. Реплики дают отказоустойчивость, а не производительность. + +**H2 — подтверждена, точка насыщения — CPU PostgreSQL около 1400 запросов/с.** B1 упирается в собственный CPU (API ~100 % из 100 % уже на 1200–1400), B2 и дальше — уже в базу: на 1400 запросах/с PostgreSQL занят на 98–102 %, а API B3 — на 190 % из 300 %, B4 — на 205–212 % из 400 %. Медианы ёмкости 1000 → 1200 → 1200 → 1400 для 1–4 реплик, и рост останавливается: третья и четвёртая реплики добавляют CPU, который некуда потратить. HAProxy тратит ~0,33 мс на запрос — потолок около 3000 запросов/с на 1 CPU, он не ограничивает. + +**H3 — опровергнута: пул не скрытая переменная.** Суммарный пул 20 вместо 10/20/40 не меняет ёмкость (A′1 и A′4 — медианы 1000, как A1 и A4) и не меняет цену запроса в базе: CPU PostgreSQL на запрос 0,58–0,76 мс при любом числе соединений. Ожидания пула на насыщении — следствие насыщения базы, а не причина: пока база не у лимита, их почти нет (A2 на 800 — 7 в секунду). + +**Метастабильный отказ базовой конфигурации.** В B1, повтор 2, на 800 запросах/с (ниже ёмкости) короткая остановка подняла задержки; `/health/ready` стоит в той же очереди к пулу и не уложился в 1 с; HAProxy вывел единственную реплику на 2,5 с (503 от LB), а вернувшаяся реплика разбирала очередь запросов, брошенных клиентом: до конца ступени 69 % ошибок, из них 14 511 таймаутов клиента. Система без ограничения на входе не возвращается сама после кратковременной перегрузки — это и чинит улучшение C. + +**Контроль — подтверждён вмешательством.** A2 и A2 с PostgreSQL на 2 CPU прогонялись поочерёдно в одном заходе, лестница без остановки (`scripts/bench/series/control.conf`): + +| | Ёмкость по повторам | Медиана | На 1400 запросов/с: p99 / CPU PostgreSQL / CPU API | На 2000 запросов/с | +| --- | --- | --- | --- | --- | +| A2, PostgreSQL 1 CPU | 1000, 1200, 1400 | 1200 | 286–1345 мс / 100–102 % из 100 % / 165–168 % из 200 % | 240–520 успешных/с, коллапс | +| A2, PostgreSQL 2 CPU | 2000, 2000, 2000 | 2000 | 30–121 мс / 105–112 % из 200 % / 179–186 % из 200 % | 2000/с, p99 288–415 мс | + +Удвоение CPU одной только базы подняло ёмкость с 1200 до 2000 (+67 %), диапазоны повторов не пересекаются. На 2000 запросов/с база занята на ~75 %, а API — 205–214 % из 200 %: следующий ограничитель — CPU реплик, и дальше снова помогает горизонтальное масштабирование API. + +**Улучшение C — ограничение запросов на реплику (H4 подтверждена).** Конфигурации 1 и 4 реплик по 1 CPU, `maxconn` 0 (без ограничения) против 20, плюс 10 и 40 на одной реплике; лестница 400–2400 без остановки, повторы вперемешку: + +| Конфигурация | Ёмкость по повторам | Успешных/с на 2000 | Успешных/с на 2400 | p99 успешных за насыщением | Таймауты клиента на 2400 | +| --- | --- | --- | --- | --- | --- | +| 1 реплика, без ограничения | 400, 400, 400 | 153–484 | 93–345 | ~1950–2000 мс | ~38 000 | +| 1 реплика, `maxconn 20` | 800, 1600, 1200 | 1161–1540 | 1013–1594 | 360–390 мс | 0 | +| 4 реплики, без ограничения | 400, 1600, 800 | 43–726 | 177–321 | ~1990 мс | ~54 500 | +| 4 реплики, `maxconn 20` | 400, 1600, 800 | 1304–1817 | 1393–1499 | 410–505 мс | 0 | +| 1 реплика, `maxconn 10` | 800, 1200, 800 | 1097–1447 | 949–1395 | ~350 мс | 0 | +| 1 реплика, `maxconn 40` | 800, 1200, 800 | 1272–1610 | 1435–1606 | ~375 мс | 0 | + +- **За точкой насыщения** без ограничения успешная пропускная способность падает в 3,5–10 раз (медианы на 2000 и 2400), ошибки — в основном таймауты клиента (запросы, которые сервер всё равно выполнит), пул переподключается (до 30 новых соединений в секунду против 0), а HAProxy выводит перегруженные реплики из ротации (34 и 89 переходов в DOWN за три повтора на 1 и 4 репликах). С `maxconn 20` она держится у потолка базы (1300–1800 успешных/с), лишнее получает быстрый 503 от LB, таймаутов клиента нет, ни одна реплика ни разу не выпала из ротации, а p99 успешных остаётся в пределах ~350–500 мс благодаря `timeout queue 300ms`. На ступенях 2000 и 2400 диапазоны повторов «до» и «после» не пересекаются. +- **Ёмкость одной реплики** по строгому критерию выросла с 400 (во всех трёх повторах) до 800–1600 (медиана 1200): ограничение улучшает хвост задержек и ниже насыщения — на 800 запросах/с p99 617–1159 мс без него и 52–196 мс с ним. Очередь перед репликой короткая (20), остальное ждёт в очереди LB, а не тысячами горутин внутри процесса с лимитом 1 CPU. +- **На 4 репликах** ёмкость по строгому критерию не изменилась (400, 1600, 800 в обоих вариантах): здесь её определяют ступени, где отставал генератор (8,5 CPU лимитов на 8 vCPU, см. ограничения), — эти ступени не засчитываются ни «до», ни «после». +- **Значение 20 не магическое**: 10, 20 и 40 ведут себя одинаково в пределах шума, 10 чуть раньше начинает отбрасывать (меньше запас на запросы без БД). Важно наличие границы порядка размера пула, а не точное число. +- **Цена**: ограничение превращает кратковременную перегрузку в быстрые 503 вместо медленных успешных ответов — клиент должен уметь повторять (для отправки сообщения это безопасно благодаря Idempotency-Key), а связь `maxconn` с пулом реплики приходится держать согласованной. Ограничение не поднимает потолок базы; оно делает так, что база работает на запросы, которых кто-то ещё ждёт. + +Графики (`evidence/track1/`): `A-throughput.png`, `Aprime-throughput.png`, `B-throughput.png`, `control-throughput.png`, `C-throughput.png` — успешные ответы от предложенной нагрузки (медиана повторов; полые точки — генератор отставал); `*-p99.png` — p99 успешных; `capacity.png` — ёмкость с разбросом повторов; `cost-per-request.png` — CPU на запрос по компонентам; `cpu-at-capacity.png` — CPU компонентов в точке ёмкости; `pool-waits.png` — ожидания соединения пула. + +#### Ограничения измерения + +- Один хост: генератор, LB, API и хранилища делят один ноутбук. Гибрид P/E-ядер и планирование Windows дают остановки всей VM: скорость хоста между прогонами менялась от 2,55 до 4,86 с на одном и том же цикле, 17 из 48 прогонов помечены контролем шума. Сравнения внутри одной таблицы сделаны в одном заходе с чередованием повторов; сравнивать абсолютные числа между заходами (например, B1 из серии B и «1 реплика без ограничения» из C) нельзя — у одной и той же конфигурации в разных заходах ёмкость отличалась вдвое. +- 4 реплики по 1 CPU + LB 1 + PostgreSQL 1 + Valkey 0,5 + генератор 2 = 8,5 CPU лимитов на 8 vCPU: на высоких ступенях B4 и C/4 реплики генератор чаще отставал, такие ступени недействительны. +- Виртуальный диск WSL2 меняет цену fsync при коммите; сеть между контейнерами — без реальных задержек; профиль 80/20 и чтение первой страницы — один профиль, а не «пропускная способность вообще». +- Ёмкость — дискретная ступень лестницы (шаг 200–400 запросов/с); при остановке лестницы на первом провале (серии A–B) одна шумная ступень обрывает прогон, поэтому контроль и C снимались без остановки. +- Прогоны сняты на коммитах `25f9564` и `940bc6f`; код системы (`cmd`, `internal`, `migrations`, `Dockerfile`, `compose*.yaml`, `infra`) в них одинаков с `f5f8040`, различаются только скрипты экспериментов. Логи прогонов (`compose.log`) после снятия сокращены тем же фильтром, который теперь применяет `run-series.sh` (`state_log`: переходы состояния LB и старт/остановка реплик, остальное — счётчиком), а посекундные метрики сжаты gzip (коммит `ecf89ef`); полный лог прогона с перегрузкой занимал до 50 МБ. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Что выдерживается и что нет.** Стенд выдерживает остановку и падение любой одной реплики API, перезапуск всего стенда с сохранением volumes и замену всех реплик под нагрузкой. Не выдерживает: +- отказ HAProxy — единственная точка входа (в проде — пара LB с VRRP/keepalived или облачный LB); +- отказ PostgreSQL — единственный источник истины, без реплики и бэкапов; +- отказ Valkey — без него не аутентифицируется ни один запрос (503); +- отказ хоста или диска — всё на одном Docker-host. Несколько процессов API не делают единственные БД и LB высокодоступными (COURSE.md). + +**Узкое место — PostgreSQL, и это ожидаемо:** единственный узел, через который проходит каждый запрос, и одновременно единственная точка отказа. Как поднимать потолок дальше — по возрастанию цены; каждый шаг — серьёзное решение со своими проблемами, а не «тормозит база — добавь реплик»: +1. **Пулер соединений (PgBouncer) между репликами и базой.** Сейчас каждая реплика держит свой пул, и число серверных процессов PostgreSQL растёт с числом реплик (`max_connections=150` подобран под 10 слотов LB × 10). Пулер отвязывает число backend'ов от числа реплик. Цена: ещё один компонент и ещё одна точка отказа; в transaction-режиме нельзя полагаться на состояние сессии (session advisory lock goose работает только мимо пулера — миграции должны идти напрямую). +2. **Вертикальное масштабирование.** Контрольный эксперимент показал прямую отдачу: 2 CPU базы вместо 1 дали +67 % ёмкости. Цена: потолок железа, рост стоимости непропорционален, простой при смене машины; отказоустойчивости не прибавляет. +3. **Реплики для чтения.** 80 % профиля — чтения истории, их можно увести на асинхронные реплики. Цена: задержка репликации и нарушение «прочитал свою запись» (отправитель может не увидеть только что отправленное сообщение), маршрутизация чтений и записей в коде или прокси, failover реплики в primary, мониторинг отставания. +4. **Шардирование** (например, по `chat_id`: история и ключи идемпотентности чата живут вместе). Цена: выбор ключа навсегда, запросы через несколько шардов (список чатов пользователя), перебалансировка при росте, распределённые транзакции для операций над несколькими шардами, на порядок сложнее эксплуатация. + +До любого из этих шагов дешевле уменьшить работу базы на запрос: сейчас каждый `GET` истории делает два запроса (проверка участия и выборка), их можно слить в один; состав чата в обязательном API неизменен (COURSE.md), поэтому положительный результат проверки участия можно кэшировать. + +**Учебные упрощения и известные особенности.** +- Связь настроек: `maxconn 20` в HAProxy рассчитан от пула реплики (10), `max_connections=150` PostgreSQL — от числа слотов LB и пула. При изменении пула их нужно менять вместе; связь записана в комментариях конфигов. +- 503 от LB при переполнении очереди отвечает «no API replica is available»: свободной реплики, способной принять соединение, действительно нет; добавленная реплика сразу добавит места. +- После перезапуска контейнера HAProxy возвращает слот в UP при повторном разрешении DNS, не дожидаясь `rise 2`; возможный отказ соединения в этот момент закрывает `retry-on conn-failure`. +- Ключ идемпотентности, просроченный и перехваченный новым сообщением, оставляет старое сообщение в истории (ключ — не часть сообщения). +- Промежуточные коммиты исправлений после ревью не все самостоятельны: в `7fbd781` API уже принимает входящий `X-Request-Id`, а HAProxy начинает его перезаписывать только в `ce47c13`. Итоговое состояние согласовано. +- HTTP без TLS — только для локального стенда; TLS и отдельная роль БД без прав на схему (её уже позволяет вынесенный `migrate`) — lab7. +- `scripts/stop-one.cmd` и `scripts/restart.cmd` нужны только на Windows: `tests/resilience.py` запускает хук без shell. + +**Что подтверждено, а что гипотеза.** Подтверждено измерениями: точка насыщения — CPU PostgreSQL около 1400 запросов/с (контроль вмешательством: +67 % при удвоении CPU базы), реплики при фиксированном бюджете не дают ёмкости и стоят +33 % CPU на запрос, пул — не скрытая переменная, ограничение запросов на реплику убирает коллапс при перегрузке. Гипотезы без замера: эффект слияния двух запросов `GET` в один, пулера соединений и реплик чтения. diff --git a/evidence/track1/A-p99.png b/evidence/track1/A-p99.png new file mode 100644 index 0000000000000000000000000000000000000000..bdc45a1adb980b63f052bf07278fd57ceaa4fa23 GIT binary patch literal 63782 zcmd42Wl$Vz8wCi#La;zcaCf)h9)i0JF2RDkJA@ku1RXqh@WBZZbRYz`0Kq-D4sOH1 zHo5oP{jvLF|7_KE6-8CoboVQdopYWyN<&Q^=NaiUBqSspMTNIoNJuCONJvk}pP~bI zhU4SnfPW&MvihFdF7G{k%-yY#RLwnI9bG&f?JTIht=&EBT%7qhdAK?G*r{zjJzYIS zxVW7D&tGu5xZ7~OXK089-h%0>VBmp-MBw@OhrIlr$`=Xg8m#zMTE{o*UZulf#e+@E!$O^Xd2LvpnRKJ#1OH1q2Y01-tm^yK8 z*HdITod>1Ts3{^hmnthOOQYUa^LLpxMs>$6>;3IM7oqOi)`&KdCdqWAk zA1?>M%uhE4N=;e?DwdBG~(Ea)TnUh&bO(6e?~lC%+l1{RKnjg^zq)q^~mg zCMPZ%ayr1I+2Uz;alSEIYnd(?BB7QeR+aoV&f)H$U9ygI%Aj`yMvT{N^e8(DbhB3Asd|&>1qSrg@G16st~|78;vg)$6Nb z{nsALc)!c_@Ulg`etdUZ=eGSt{k^r7=vp8l0+}FViYnwT{{x*LxYh6M{q^;=l~GPY zLg-Sf&oM zyCYSW`eK!gCoV27ppnGVngj>+a^1?|c9K_Pi&rR!+ly|=hr0sG=gJ(Uym-2`bl>J{ zEhBH)E?hm(<=^Dw)H^K*jIAoADh?!-m$O0is^e-c`)GxQg=ZW#S==%OoSqX9eB~56 zwd(KcqWQ);mc2)=96DVfj|Y0$H8)2hEiLVfxR~K{@%$zdaxaFCO&qF}!rC=5q6#8O z-otijJ3;@^-3@bCQD~4kIUV9EKU7v#<+q!n>$XWHhy@k_pWpts23&J5CFG=s2=ZI$ z?q)mJcA^kdOiWB?s*gq{f|`eCd?!H3F1M}vfNoBYVEE!_$p|f=H%mM~7v#pORrtyO z;T4DByNkW4D-cXvM1*v`FJ8XY`%q`MphA$FTO&o}Xt8;62TnX1SyRL9)wa?1tEUHz z-(mVEu-hNt4IvM~p0H(;Ln^=AI_n`wNr<5HVt<2anG$Pn3`vqf(5W-XwQgEZGU)cT z|HWR>w6%*%Zsq;`e1pSyvxf~TDr)IrLg?v|&x)*$j!q6->oS*I`$nxFFi|Op4{_W> zcOaD&wC!TrXb<^QAkhW_u#cFv?#S{yNj70h_bRL#hgm@bXj*M;?e*Q+m}a>y$IDe; zKfkguX1L6>7uN=P%6?oH)hWe@a$sQFypQ!@vL;-sW6~@H;gtiIo#!1GDKYNb2-*zb zQvVuj5`VgX2e#L)gWe;qRvgcYSW|`tFGHo{3Icm0X>v73*Pgv(Ej?s2>kyN;-J#=O zIW`$$;o!j0E>V;H_I~7dZL&iUU^7z<3J|F+ulu2Ek1_F*Q8(dSQhxhFCAm|J!4=G# zBgKIa71~ui5tcMF3FUg#?*q8zbh};jreXf00)llEWc?9MQ#to6aZ+jC( zpZyOj489#*`!R>qqDBo5uB@0LQa&RP37<cQ+A5lYVJMPZ2H3_cJ)nnB6h)*EMCQbFem z(jSBIF^+m#THt3=!SF*tuNU^$Z9a4$>`e?A&b!lOqbCH-Wyu#x;-^_tA2#|lgJU+g z;K5{Val|}t5Sb;p!G@rbILLwho|T!AwT5 zhGn&-Eouy^c25*3gMQITs^O5n?hkA~=-V2}EblytV#?6~VbQ+>dkvhogQfsx z?%ni)^>rb-75ZUL3yo!Da^10zyYGXwDVEmXrFpzj?1q~?wtv;zX@gX^J`$N~Y=6A_ zMdtitgAo*$p<|Si8AB%@x4Pg01s!J7>%VTBmWr!Q))?$apbE?|t+R@HW=~(5Vu{FG z!H`LSI?nN47+8zDYg5DJoS$#ZD%3Ctvb}K@+X!HXs{l6wxf5>cLNOwoXKU6>uGT^x5GD5#r~Pz(9de^pH^c&-UwH%%sD52^9O6H&?V}t5@q;K=`)_P~`=67O zS11*I6Cg~pPUAGunyCy>)TLQqSYTRl!wlXOvGGl(n<_k;hq}im1Zg+egBr#*A~fRG zf91#sGv^<%h#+UOaSo%AcE$JE2@Z4TyOUG?8|HC9aEOUG z3t8z1%9yI#5Mgu=p230Z_g|@w@?jco^!gpIc0s0p0x2AHOovIhVLpX0#)wH=XWLJ6 z_$Ny;Vm8vhAs~bD7x6#l)xrS;)4gk47Z<&xRy^=?ts94 z`c!YF(nv7q&6_uk)$=+D9sXCx|SUD5H&TV#_tDBQBv$85&2HYKXbX0}+^yx1u$zU-L1HeH$fUxav zV`GCD$rR+Y}5gCn6&vwr&IUs*K0efaGjeY1m*qgbjNbWUJnpoS5L<9%>X75-J_@<+8A} zya)~s-ZKVLw{oNPV{ zJ;q5MBui4E&t{27`q|`r;_|E`V%(OD5=S zHJY|RtgS9ylp0GWcvQ7rTRRpE-A;3?{`~oK2N03wmSy5H*WirsWvPw;UKb#JHcNy| zJFI2*LsMoXZ+8lAH#RngZ`>($D-EZ1yarseDS53F8qZ{U;a9Ng)B&%~*&9j!^BqT6 z!~2mh!|N$4&=>B@Hi#{YukJLmPqpN{wbmL>q#He>#v54($KE*>W(klSEe77 zn%z>*?xdYLsW$HHCyQeZOK9FDld*Tlre>iStnBZWCQjpm4#S1}lbcZ%8pF|X>K z$o8d8J4+gwY;(sD|BC)|+8M8Ac=ryR zgtAICgO_1Op#AsI`}pQvA5D{e-k~8M=4f%uaxqN#5C}vXZRedg^;RKS9H6^le~L(jwrm1ul4WI z%9euQEV9YKU$Q()qdm<6U`i3>i{gb83vL>#yjGDML{VW1zf6P%^u18E4`DS3wtrv8 zRk#RP{r z#A4df(M>rXZFGjA>{Zu?3@Sp76dEMUp3&xBnJ@S~Ei0zk_E9@+W)Sbt!B3`Pnxo3k z&xeRuPTS|y#cF|=nPj}EyWMyz)`L6nA|N_io3eGiiu|w2IKd4^?c`}dM2ppFO%h_AC@FG$n|SqE`b(8S5GKc3PV*gmHj+ zW(tQ)pwvSK;-`2)mGR2Gf*v!**rtvVyLsg;bDeWp#QOs=Ufo#DQM|HL`V~n#EUTD$ zHXd)Bv~`yQf0{l9hk)+8+w25l8D6SESYjJA3!{z-=V)6jxsk=aGHt)C)}d)F-3EcX zd?<39_h>Dw5#AXF13t2GO~x)8LIdB|h=+UzW??ao1>D}ov4~L;xWvjxdro-7I@?)n zPzg4O@{ATl@$qgt$&Xu#_A@n+jopKVf+dHf$i%BRdkJINqI!s5%Gu7%2?O!T#!q3M zZaiDT@Br>b1;rKj zL+_Oig&&Dtpm4zKp7lDQ$VBJ&xF9ETUQu*>1vK_%$H6=2|j185cT5{9Z`) zCOWmG$E3o>N!!Gj)(<~u6S3hB?5Gfw@Z$-LYvZLP*~)|@dYHmxFQKpfp$k55E<>it z$eY7nUs}58$FH4{OuI?0V9+={+qhN;3c|GA2*MHcc3=0G#Kg7|-P|(0#Y_|+g+U)W zzHm9E7q+L@Bexhyv0+D%6mn>qDv};;F9P9NYX0YduqzaKB0S4%Vrwl)T|(2zaifJL zq^LbQkjX0J^?h2S{Dia2vRM@ip3Jh0^Ok$#=D>FrkVtpJ0TV zf@OfPi#2ozg&Wfudpp56iNO`Bhv#v~?X@BcWCQM$Rs=5bWy;!OVxy(-i^Pq^(Ewqe zqiw^Gk}Do>T!{mhEE@cYY=QxedyCMwuN0jl3axf`#tZgX3KoeKL#@7dB{&L3l$2lX z1;a&oDwH`rujYk7b$bfosyDTXOfeI-+9F#?axnMsQy`@;o~+o6=r9@hTXD-h!PiDp zD1JAqcbZWYz7i1B{`$EfWA%Y-uR^$kpF0~)q4k3e!`#$S3H@F$iG;lAf2ppCqg*fcX@De5eroRadzZc!qP7mVB!a z?o6_=TS0Ebk`ErXai_DZFb`3Sm_TlxYtd!_RvD!?hfs1@_!#h?*VJiJngH2BvMI`f ztUaxTX~0;-Zm=!REKIb~mgHbMN1~eX62KN9SvswnPB`= zN=X1~Z{B>bqxG*__`oT+4^XX$)tazp2er{?w5Bn&XrZZf;GK#m?0*RK zIr>Nwn&!E36NK9@kpL+LTXR-3jTK8ku|Y6;tbEiM_rqi`UAIIM#8(SKkC$DT%wO$H zcw|Gi+4>k}E)hw4RuPWw2Fm%WERw-Y^eB0OYGs=Vd_rJL_957Ipg>`1ntid>m((Pk zZ=9bT`nruJReEM!GQ>V5vIj@}KEQ)#8vok39=uYGtG1$OTUin6zF=_>P-oT0PUx?o z_f7cfZW;8X|HUhrl*)?i1x0?0vKJgvuo#eJ0u}-x!T(xrVoGTW+O!h9U`y&g7g0fE zuh^d1pac$E!<=Y8|AaUjlPrg7#VTm^k&+c=YfsNS^CM*^(kUCX#GP=7T|8HKiPL;D zq8R5gO@(TgMl-dB9mH+TAlP#C;!ZIWxw)%<9xr~8PzfY&cLmNZ)X|0z4fwGO1{x@W z$ZrC*K3GIw%KOl6mnNJy_TJKg_jH7t#-x;3zUg`;ktxY6JePsK#XAk+E_^=fM@Nl^ zRG6K^6R{`SJ8C(X*6aLc6VAC1QVr6?J}f_-r@B(El6iBPk&pAwTCu*8-{vCJXrQ)i zj7z30K}R$h?9d%sWS$WSEnLdU4}nx{yjQDYxFT&+Ak(BlP+B{|nP%Q7+GHi6x`GFd zloSdHJN5I}hbS*w9rji*yKl12^5b;2!LV%9iYpHZtT};ZT}3k)gB5KhZyfljYp}uI zMK>XR!r0ujQq_t(uK2^S1dM}9&=*H5a1gy)By@11id?nvbkT` zi|U5F&O|Fn-dUh91iQHtk4)eoZa>K>4Z`KF`T9KHJf?m>;R-BZs1>B{>`Kq<_bWI- zMcNs)(BzX&W@09fp9Ke81@ghT*@hgD-6wkN#6MGGVI$$^|3>9qtYL+ZtzpVl)_D$LW4LE&$nMLx_%;psS4G( z6}4yu`6fcmrn&I23R_G@eA86KN^ss=iIVz@#4!%RPBgz@5~Y|8InB7F zK&CoQKAm;`HqLEmdv0(7y%d<9+}+t)JbvSrzE4T1uCS2Rra@mg$2~R`Rzm*5Zj5e$ z4Nr6fF6z7}1meBRQ$dK*zzuJoC5^GFUlh?RIG7@xXq5G#m+ zni-WIgn2oU>$8~LafdF{JDN^wlnEI6I}*Q63PNILBB0VZO zS5>h!mRmhaT$~gUW>Unc&okD6j5u;nCB@5f!EV_N0Kv*+T(&++4q*gwJ zR?r__0I9ZYB7P8>_`3w$==Hl)vw5O;xiX&G(oXzaF;VI!xK~@NQ|aN6;?xNutTcm4 zoOuDjL<{i_pp@m?^%D%|RT zk4!6J5WeMEqEmHh8jgXdyQ^9x`ARrvf++gjtpR}C#?@&$gJ28LM6=TTAprEsUQhT{ z(pA4+IThJO0W!W;<=_o{A4m0=TS<^t2?CrZ9&lOer3BS<=TyoSa_1ygW!wf|+=hS;6Alv*wA;S{f+F@vS+PVE;LU>os$uGe10koHC8SyOM_|hoiWeb~ zf{sgGI!(77eqK$FXi_k|ZHwD{q`F5@c14F0F3lvMg%N2X3twfiE=?n;HyUla@;9dBDqi* zML2T9V?dZ0u5lZmxDpW{{(!UeWaA35?We7Nh+D6M z2elT0grMm>X%w&Qgb=!I)9ul0P)pFR&#`IKz^p{G_7@mj9HWR0b1uZ)c}SQJ8o3e- zue@Ju_VDY-HQu(G>k7w!)m$+{=LQ~*nDS_8Y4>fIDH2|fe~eA0R)|VZ7E{6$*u-No zNd%|jNnA^TuIRpsCJ_YL*{~OFXchc39O;zs=~Jmj;+_oHKx_VFN!?7vDy!C;O-okm z2^|f{C2Hpgz_-QoX39R4R;G^OfO_^Zj{apr;~T9H?|1IfDaB|$)U$vu^(Syu=ff2i zNbO*BWtR*WLikJB^?Et{4ypD@a$7yD-oSGj=2aH*#SF2dn*&XSQK@n};Cd+nKjId~ zcQPKdbpqJHNFFcV8l3+(YMot&>Ra9`$K|$1A%j(w?ysEp(pbI~fY8$W)0J!c`*00d z57NL}dZE4fhVruIxWBJInmpYc8ZS~N62nGUV5DV`j|kT)(_%ws>pKet^KL)yPhs=nn#S6y^&-w=2-s;p}i{Sztb-_5dL=~4Knr^t#;SZNYPZ~ZXw3&Sh$X1?3N-o^B%|5V*2 zkf%l5S6Zaqxxu8Pdw++{|2;g|H!`;q$RsLGet3kqNR9hHe_!A(`+PNn4}cl?Jbv-CVKNUiL&TGrC4s_>cI%Gh6h|#p}Cz$kANya8`4zX_7IvaNjybQYQkr(|08{ zq*7x3r$l?o=hqGp-}XuGOkB8n*Z0Ay1!SlGn9h1|5^5%{9drDNXMBlLDZC}^;-E~KkF~$D3Rgu3!gqJ)zVatIW)xb_V)g@e=$o@N0!Oy{2}OrIJp?b zuQDyB-{;kK7Xje^R&SGlAp0M4EIjFIjKwT+?!%Ah$F6Qa%RXuQh77@lh=OoOUFH1x zPQNF3BSUlUUgAUxbP85>Nb|&7=(fs}{HwZa3Y4{YVrPUP;h(VB-+s2S&v)nH%8HkW zzjuJs(QqJhO$B0_4Els=gI`#XTh{Cx(6URm8go3fpTU^_48~df-GF}dz}8E`OOt4w z^@*)gBgBM#;N9}cC*|1g4eunLwsF_w3r;K|3?iMg@`y2`;5fX@;7>!fX(wlix#pGW zg<*dUj297b#~3{CZW=qoycVmTJ3ar>B&z+x;Y?$JlZKI))*e1Tmu%XHL%}AS`-N?Y zMC3Dxmo)i`` zLjnNg!#!doUX#ASQ+wMYVZz{=uvyDeEA*m_!Kdbbq3`K{od|YL2ojH?oxMxoWC89V zny=MS#_w~Tit(OMybth>I!9E|?g=6(>BtFQJ?*)}kl z+H2xwJEZkD>L~@-A0}L`EXJ2^Fsq2a2C1gGvW3Bnf~x0fqk?GQ?=VD@PuZv7tdHG`PAYD|YrsB;!nbHI$r781YZ784A$IEtM0aLQO zw1#8_S+q&CwdcbV!@IA8NQ5rJz4(zi1-R zi0?oD{P-&uUiEIgqujntQCJc?q#}BJMo|P7-xLxP-h>O;9o-bQRQb%qTI2)dk2f+h z_{79df&4N1q*40suCQ9Eef;@n{biTESmz{3Qou}PLHNn9cZ2yX*V{iN?c1cs(XT5u zZCHhPYnqxW6Mi&G;QV#nB~4>@QuNV<_(GlC*_Y|p#zBN<-u$=oFKzTzOXwuaPIyhN zk7X99cqjc9SpPTHe{|Bwn$MX>BYVqt%@51-tc#Ees+T-0@#J9SmAyg6f!qzrJUlK})>GSwzV+@%5%uL(=U&?dW0teU=B)VPN$oEf8?!&vEn9061034p#flnd z8ry5b7GHkyXSwP+Ur$_UcEwrQ>UknznDhhA8^`F>CAE2S1CRM5c+bC4xW8AHso3Jz z9YSHes60;r%zGWM<*b~XocfQTU|W=f+syCtJpb@xosT6N>HmGD?!;RjcJD{5M@uku z9uTi5yY3#`Y?@Z3-Y35x`D5)@xBczzGxhaY5tqE0|5|*ky5BpshI;k6Ku&O{K3hn! zty46sNeGcRa8-fUW25V6{$FaED@CNy9#2U!P5JK9I$u}Wh!;qYP4Fr6 zob9Oan~t_-DQzDftj7HKFU2I{M1NB6O1QyDOs$Ii7A{_V?uwN zP924h+`9aUI#qX#=aZUfI~x`K?t4OOVPU}?aOnu>oEU3sYhu!m@92jtMl1fv3?YBh zLh0&O-}jq{v~y^RMNb5#xgfV@JFCgR)S#u;W(WuIIn{2OV-kegPeHnRWcQhszwRd| z6B@S`%TE4{2h-H5X>s`>%QE#DRby>t!3|fNs?b9KbYR?#R_sP6Xw2#7itur zHD(~OaB|{4HW7(zkTB)icTEY7YAP5iy@eTG4Ew#^^VIzF`bw? z2WWiWxBeL3dr}DWmIPkT*$BD)LEW3K4CtHbIG<2X=Qe+0>br)fk|l(Xi;KIn*v!}H zw2->r0LY)9t{Fhcj*O!a<&9>1bd2)aliwUFbG~Y^dK_CtO7ZUVZcZ4zhomLF2xA0y z5rWWHbeq$h^`L@N3T*WZ=u+t{QcmLzzPlDPYqA-mWY@1jG7G*b^3SHXn5{8K8BFE) ze6<`fwgXK>f&l6otHLcX_p=l2=a19N=)EX?+tz-ZKzils-I)yR%g7wfg2U+BoG#JK zl`ysH@Xz5`j18IpOqdq=J$0m!0yn+^bTa7v{wpriiYJtqG1$f?CMN)0`VN2_-`zKr zavlVLMiQH`9R4|h7muIVj7+Zgd`v}}m}P$xbb*%Um}51j>Qm5e+q(98!;IJ|+8+9H zGo!BgdoSdPb`jrWYKTtRS1li#sY1mU7gxV%I0gX~uJ`Wd40Iua8FPPkrtY}-9UlAc%(y=q_yC3mHbVt3ZAj&gq z2q)OO{xuK(Ql*99dD%o?;T6OVDEsXO$E>~~YXPqEe($KF@JqINa{>V)3r`Cb_?J0Q zPs^n#MkNcwHNZ)H0s^{d0da8x0lrSJ{sj*I*O!-|<(dP&E(31-N4p21W0YKkx;2dde}BT}jcpvtygoo;I>9tvu+gc$TdVH0y1hVlWh` z0+4(d^x=1dgC5X?*157mwMENRmK?44%oANE zOSSWD8)w)>+MgXhM`Gh)%rp ztL$e;7cr3D_|uce4q1UvUiC8Ih z3(*TvIplsBOc`9WX}N9n!)w72jBY36d3$aL;C8D_Xu#Fc@aDg9!>T9WA-xG+N9v-{ zO9YeD#xFTO+VV-PQjaVTXA5?Gpi}af@a>W^f7~0O(NQP7HaFy6*ds@$8NQij z_Sp1-ine)VkPbw&3Uvr4MF8j9cMnBRFZ7C|9LML3YKfcAOM&Qp^bb{lU}#eJ;lfSk zi6D7?>F+hUo&1ciRE_0DVSG2mImsyk_3U|9LPAza&SWvRnZm@0i zT!uOM;W5#wmF}8?0@iv+xxP3BRAz%DbQ86xg*9VvHV0FNQYQlLu1{Eb!mX+ozS-bW zhTOR&IS&Eo3bYkx*q|S%`1Y-iM&X9S2fF;R+F|bYwG9rH70)<~9&pbx>g5Q!w;KBB z9oA2~BC6!2uR(e>=b@>A6@F+T*!vm`3#TXWUdywo5&JglvefIm0gS!-6coTrL7Mb+Q$;4>vK@`^=j5T{s?XOd}uW9{(iv7Dd83 zY(FhD35p_5gxnN|!u5dj{Y#Y#nFC>Fa&*@j9Q@gu>UP0%L%g6#f5kk6u!ws-!Ge~zpvX><`A++Apde<(AX!* z^QFjh#LByRlXu9x`kuSy^>AQkd{-6>cf> zT<#H%&UE|ign>d27)Kqeg!8D}7AY-VYe1lQhOCm9t1IPmeG1z%WF3*lEW>sz!b(i? zFL?A0ikQ#n$|zfIcX0TZi|wh=8>`Zr+36)vWv20@EeyXZZ6j36T{OC(5hP91jWy8} ze^>6-I&kfeZ8d~~L}&rIgLC(LN-|>3C9edFG=HryB5av%;zsk%>ID3ym zu8u&Fl*bc7)j_=*La6d)*tk5@_fnnN6iP9VN>w|~;U#iwD}H~$vv{j$YE z_upc;f>FrdX59SFOMjnvd9nxJZNP5s2{nc&Pzxdy@7O#p&D~+Q$KGD7*RjP1^xOmv z5=ax;d^RO%_W$Sz{SdxKKlobx#DD29gl=6kt$yu8>YxajO*Of6l6f2Zsu8+t%TflU zNLhWx^7?D{_g5*U#ge{XNl8;Vz;led<>F@M3k^>8End6RzSSoIkE;WaW2kl=eK91K zUkapwkn4vo-~Ed62ZYWrGiHyW?PPx_w$>K*a!Qwuk%q;np{43V*2c7At6FVy-$+%9Kuxh zufj@)q&8%o5~x{d#NYuXKkXGMthjr>`BE0CTO(H!gtl+K9(eHE_p6XByFk0}9kKA; zOgzmP3KqR5dzE8Y+FK3>=^e(AA5aW8Rst{vj0uo8KpoaQ`bY|#maP<+W8qp0=vNy| zkU^mnIv`Qf`3~M?TW$P+(T}vOyyP9ScjJW?)BwMOpPLoY^osmP2$S=l^ARD>>^cfb z=GAIhhH-Zy;YrIEkcv;IBMdX zROJ3$*hclE{o?rQN_+B0cMrbgh0~|UB5KTNluk?nZS0|qY~{Q(9N4IgL%x?^I^w(^ zP!F;y&=k>&NFfAtSvPpId?x$MjlB;@E74M(Wg^>ELLV$REFH_W(7gb9_I<&x!mp-D0!E%QRBTspzdv2Uk z+sqb;O0ZICr?$rO`rQ!5FBv=^c^?%g$(KE1(HU0lR^QGHhnHoFuCN+CWHuW`FYbOwg8Cq57H|@<$TPV zdiyJ-?(!V@Z;EiG=C(SIc00!}{bkcMCl&jx z$-vVuoEx}>V9PqFsZyOY`9wt4ft5+oDQD2>)3%Z^YU#w}gyk4T-N{@D6O;xeI|jX( z^(xE$2pWY6hOXsN%5|f7c@PwG1vT60_!p*r#*T{^mIN?o!)mFs9o2BkD)V~TEyBM_ zkp$$l8@qBv$H;<{B#_n9T;m%!^NrIudp2%;4NqUJ#s3~^TGiiv8 zCico}a+`vBNJx;wd5_0b&QQF%s@#98{VWb05lqdOl;-(NwB(^1LmvI*6Y*L{4J;72 zem@Wu*DUJI3{{imduVT|=EMU1J~N@#i!cQA$wJ4>{2ei>F3Nl$k`By5b8b!L4R7xa zx*ZB7ziD)8h8s-lNJ{BRam62*H43U|&WI-}0~|adb@nv;oSjsn?ezr3jcUlmIU7i* zTYW0jv&FA!FC|E;3}z)`Kj;I${ox&$-`r~<1C*NZDNdQ8f#OPA>QRH`1JoN*Hu z!4xlnbrx>#7X*0;1Vamc)30_KzuWgQd@mFp%4(?OY$vLtudS|}()@L*p-caw{kmp2 zPmw5?GSdh4(P0skTnqiE<&9qxK7DgiEn%t;#wVXrQ0vtr!uzpy9(DlR{*ygEraj%L zzUj;q@9NHx{+3++)f1+fC;n8OESRNJAmtVs5J{ze7@ETNSgyRgJ1;~}(0$?&Og;;6 zJ->BO{Ll` zWo)Lqo)Ymxzh}P)evALTwb)ACmxey$ZNH`ycW=2BrBm)8a!wCrhbL?}r1Q_c;hwk6 z!1>Jcrz7*)UgLm}$@uImF@#@S;(<}lFD6$_lJp#osJ zVDu{v`#lfB83@^6)uRHn=i_Ff1P zk;ad;{2o>zGA-TJ3X?et3bX96}r%O zVwRvm0)^(~BFJu0cISuf27+%TOi!j-k$Iv*GbvoHC~J0=-#HVlHsFM=9L70UMh%SZ zWboO11={*ECLLy~dVq5m$G7{{1web{^-e)N_xoXDl?-0AMk$e~lrb3~hciBn8-%Z= zrID6P#5_~#`_gl=O*V3}mlgChO7RC(0Oz`vuUK*Wuj3A_=mq0 zl+IE$uFm{~qs&NG23lkoyGTXvCUR%J1P!3D55UkW=%LfO^37V3y}M z#JaGX7pUcUSv5fp3!e{d8V;;SSg*}5m1EbAjE@-zCSb$SG#sEB?+yFc-dR}kO=SyH zm&sVa=EDh}wY*;4p?!BYGe6Q0R_|bU!KqO#h*A)#VnUV8jEtu9a_khD@{?ke=58Au znD+Ob=7?NAfHtn?uCx`Mm=}Kdw(?-Qe@l_D#w^yd_y(2Wc|%sUZF@ye&ka7`(o$6y zdBUlM>%CmGW~GPfT8?#1IHT2)=KKVe3!RdxoFTb}ml678w+P?x6`)JX00$I!ONLU| zN^{AJd1pRJ|M6nH$M79tKIlQKknXp9B+3Q+N!;kx;+yiTo)sq7Yx{cAkTO|e zF@q9REw3_Pn{`nksvy&mkmjwS9fP70J3V$xvM1%XGNwV2iY3jxO1WRFw~pY73R`dN zl2Y+i`kUz5RXp+@NS*u|n8L~fLVrz4WfV)9<0Sw@&j;(F)U{Z_WsRx*Dl=#tQ;sjK z$IVo^9-z89>`ejsZQ@Hbc|^!k`Gn)=)Ra4(GubnAdGtCQZGXW|pNE_u$+6qdQxc*( zzIj_3H_iJtCKmcMc6IK|@^Www8g34j|MQBN-*3GneM0#99p6rQ<+U{ zUoHHFu0SUjBL!P-*97{>fA;m2?|H{d-urHy0W}4RT$qF@&dVoaCtVYC3RAl$%YW#Es7T|lD61i4`r<0KnBIUfqF|)OmB?+uu$a{akg=hHEFBY-DP0B zRNf|co@Pkr+)|d2ZBkIcKrkqD~` z7>;cvh>xqE5>|#;MOqx6kCjIFyOFuNv1ycJteR(ow9hsd5VMx?Wi(8fajBdpU#Q9- zodJtc@Dta_a(GZ${c5RKiqxG|S65<~)s&IM_aX=Z^DcCT0veSw=>xgc>IteS{YQ5G zOV%yf>@|ZcpYn?E(Y@4AQ)q9syxJqA41z%c2awz#0mWY`8eeB2(%n`lx`YYQt5| z1nbWyqOD#zoG|{fLR$S}H_B0OpnPf8^Pydj7k=sAWC=2kYX0!3ge0F63e5}uX%$$> z|LP*EpQ1z&YV#gx2m|@^?Gwr(qz5^v9b9T29nq#7I0~~N&NQpplR}^$hT@yeSnti* zHa-bS2HUr6&*4`TmDVTJz*;W6OiGRp{&uyxy2;GH)M2Qpa#+yNRl_*p{Z=bD-oOtV z3TuhULooIc?0`Cu;HC&BS5d>ZeKRo!LYYQcM|e7}LM>E<+tE2Kg3XXNQvS%w7{m4q5-r5BVM zO=;da20Td^3P4}7Lc1U8%xUnAwJ#<_Bd33?unJ5&%jWRUvdcTX|xj;w`r{G zeV{{O!UzalE_%g^=Ktp#s4p=c(oF`X>I}$+SiIYh-b1dDMU~qYuyu;5I%*eR>5+Hf zn>fHJhx#={yrrKwQW(V(W?+=iqq@(*B|e668^?_E&b4dpzx1r0pwJyDC;oM7lGnpc zEk>RoprF<`?pHH_Qb@x0ezmM5N2{#kuxOo?j4AtP90_0VVkQ8Ot^B&iE0!`49iA1M zD%LP}@A{(v6wD8%_6DY)Rx&`rpK9F8fHJ9zA|<^#iVIl*(hNv zJoJ4ylA!lD@I$0@JYE_F=y2+%(IH#;>q{!9ho^AoBkePh)Wc7LWNjI+g*SM>36h>{ zQ6Emiu0(*r=aO-)(T!08h^|>#n{x`Qr*R@v>~Sny`XR2#papx`THkjb$){7i@f^z` zFN5A6%DoL#>T-#(!hv5XzH?4n=t}&?biDKQ-7J3IVI+3kv;I5rwAw$vy!h`Gijc6K z^hhKrZZJc#;mRZBFZBBWQBhlfQBp+9|M85mHuT@XSo#lP8@B^doE56P`ft{_f%g6K zJ-fzYLpPU{wW?r}N-Mcr?>zx8(b45-wsf5D^)J%T>R*wY%vX(gxX(;@LA_9IA8MzO znO}Zdbr>f;OG_1LPjC9Hgz+%eGnMXc8~sIZ@EajxLpG|OJlkNfzCy;GJ-^?nm2rzF zrc4BOR~QP$b5c^st>mCEP`M^LMa8H~!$?~313BbWB@J5tvCne;+?6PumxgukTS;=S z*HrkWgD@xWL!pz+k5iW-y-OsI)+bQUC!CJsTZ&X7;6Ujo1%18xP7i$3#zmfw#Orub zcNf6r%&#kX9>IvMZ0^OGbV9RmkNK`Pq%Vx@14e_abJ$oJh0p8WvPXhpmDSQnb@Iqz}* zLa3fwQXslN_qQgqpYxW2e3P@13qSFK9(KcbtSZ0kmT*YOCK0H955V#Wtp|<-o*ym{ zLGN$N%T~SsG&{RWI;uRM_xf*Dz?luJ={pV$zJDmFpci5{`H6$FbtI#2{Or98H1zY@ z2?UZ~G8DAzt=LzU+3`dT5fqRU&i5;i>Uc*qv`M!zepp^a^8OU|!rqv`J8-UDWO?~D zahfGpxgW|Nw$zN0iQlJ+n1O0jH8nNJ`J-m(Xk_ci>pwP|LP9RC4JGUdXf(~oiNhvM zX88&h)L}a+Di+LY`$#$Iy=q^mi?(0&=0g}77m{Q{3kCU{C^C*2dITj>&RuF(;zu1e z3f>2#37{v(y{0?9eSf_*Ja@g5qtbGjX;3nZ<{~0JfNvK${qEW2qQ{syK!O(s8v$le zN8MzRbbJ?MA{3-OG&|icZ5IPGxLSjN=MpAl7avRU@kaZ+FR6QxCM5@*T|D%@_IjST&ZxPuzYw}NeEJ4__LiCzp@KU_D@E?%6!MqE-aS!s*0n)KRdZX z4!(tGBezEi~9QG=<3r=kNVK#RPC=WpT6n+a`ble=A2Q~$@e8D|^_Ty*ci#6KApZ%;E zVp9IV>G7!*;7phK$QOP&z95CyBY&-P8rT^@OQKO5RQYyNw6!4{`|cbDc{#FI+AxMdo(nyDNDc#+rlyrB9fOKveB&55$ zyFxkrW^Q^s_Y&(|I} zzDmb3=d|a2F8gu?r}mFWI~QM2R~GoRWIcF2iX2x&bwX_ewV)SH+x=N+!72p$Pzp&n zLd+c;1!TbMX@l)=r~8U%IaFX|F!M(nGO$@-wrv*Cw73zTFV=Qp?7k{1hc~~1(Q3m) zP#NvZlEBoiIUhkpjxI&kAv!!&oP`NndPD-wk+m`oT?<xKH#?m={hL)BA&WvEnDm2K`@a-GA`Bx{>K^_1L22D_0 zj)QZhppb>JRh{Rpk=N^OlXGqYL>ELcik)vT2BPgw0$jHCe_QtmKUU?A8A-P_02;%$ z0%jprVIPVv0M1YV^oCe^P3#=0xWT^{npvOL_x3_6jgQ+B4x1lkXK3_R?&S90_s-68 z&YmyVGPMaRk6yJ-Hd?K--Q3-6-{ACW3C1vxP#dPxZY#4xzn)Vz+}4fQAMp_{IX_3m z7EbDIABNb&lHgq>ozV%iUM3yQ!hg-JHX^%4ec__~{^at5%hw$*U)#Y_0l9P+m`v<( zuLX=S^Q~SB|L@-E?^7GU54;{NG{a2=z~OX2nd3xh zs57BvG&4dUV>S!>bThjoNJip&Qx&|s6I$NW6 zxc<2hX%gOHY1uk*C_Rt3t)q{OOfR&K?uhOQLWiyO!aTS`{6)E%rr$vR%Y;IK&I6yi zk_ow{z#kQsgrN511w?6*#r=<6oPaLec<0J`DWwGPkd7Kqa$)l?EcK&B47w#KOTI#N zur(_BC(-ia{P z$kU=c$t&a4ORB;LC;91OsdPO7NhEYO9GPMqmy7d$wU|2?J|Sz$wJ)GT!_5goc;$53 z1D)7y{5ITT$i8nEe+e$(*j;;9IQNHYHa0H8UxjdfS6m*C@AQemvh-+D!SUbU7P#>`red&dqEq zT{^R!V-$K99VKD&qY}uh<|{=xstc8Jq=d`B)k~A&KZS7buugjKbi^b=C6f8WkXwaD z6PCa5LdcAXVN&ChJuRPJ9MN*njP?2I7hSAlm%>4FCvHj+7+xyarmsj!H3C0l?IX=8 z#f-K2emyd+Zv=>zarP}f$NaQ>n!@Gr4nsH3x=-Hgb*3G68?s=z<0=I+@#*#_j-x+m zLFbulSPA*Y`L7r4>G$xbl{{js=uWV+-&-wYh=min9yM&+-k$IJ>hXi(&7STzPMY2$ zJMg82zr>qi=EqaA`efb2i-lR)l#0ghUF%jf|T+tdv-! zxtvUy@bC3W`A%&t)sG5}rN5GzCVO#RQ6{L|d^##odjN~mE zYfyMN!_G9&IX{h7a0f|*wAYoi^=$kBf`_VwjcH^qW%BO?xwecxeSYTMH3~<@)mIpo z0V|4DW85ea-YYo}_#ngRJForJOWiq(}{MPJk!_6 zZ~DiG;kOUPRX)&pFS+7vcl|!m9doyJs9koL_@PGB7_jQ1tF|GG%ei4bXw9*b?fTMl z;a=IB2^FP^%8c1KP!4>cSHL?O2E2GT`B&c%Lfsu#w2h>SqZZTkj&Qs`M-aHU znX?Q&eLBOYpv47%hz|-13R}(lvt@?h>6DO@6T3QE9tRdC<+6BSq)S^0YYU#6(r{#& z=4X3x?frCh{b!CTsm#vF_IrBK42}0~^ez{%8zCj1bVjzgWb8S8uuz>)5tAy90~_Ah z*j#uy7Xz1j(?{Iq>7T5qHJe;uz}y}&AoRKW@TO<~3#FJU53Ynltnv*ECN71fD2a->`SMLKVfNOv;DPAsBw!#6llJFfd3=VV)?>TXm2VN1IMq+b}$;U#Ko+)r-22xZRr2gL?XVc|8sX|9u4=!9wc` zWN}&)sFC85)W`p1@o(rf5_GiUF+u-LQBB}TE^)FAML-iPOz=`iae&=jZZ#ORKYfNA zAxXgM)6w_N*KW^*{y_J`nzFNi(#5W=oh?vZ{hF->&+|GvfTjgZc2I@eRPO2V?zj-a z4A3v~HLIu=>TDvxE&VS!P!+W%nuaQEFjZvEECcIYlF)H418!B(0#&vvps7gz`kmVBjh5~0fNK(JSoq?gHr$>WU z7(X5X?&74MJThO2;`DbY(`RTR11dm{^GC~(G^y?dXm(VcEHuulQ1PCcP0A~dWs!hv zi5=T+?%wNyF!fi(2esIeW|H+JF1m4a%Rx=iH&kWBQfg|J6KtZ z>l?qb5M=|f^NDg4Dwn3LzR4{GfkV=zXlnfTgWNQ8xvVG-Pl06nqy9Immp=6=b=$}) z(^6sPjy3u1H1D{n^}MR#25&spK>e#4bq<7O)qo~2EAg$$GPbVEfgE@D4f(+~;kd@7 zgsSkCnExq9>SBFaudr@-1~V`_iw`g#<~mBQz?g@dB^KV;@=z_15|$Jz z+G$riiwW*wBa+;#l;zh!@2CTg!;|y%ztJS1qi(DgxL=bO!krY*! zn=`e3G4q|COG+9EV;Hg48@b#3VS(l`uBJWTgtJ|0A>Jup5iNA}%QyAygPw~W3i>$@ z#Dvb5%q8;h*NxiN7jn|rz_}y*7XvtV(5-XuKOB3CjvVVX@ZQeu*&L0KYokVKRe$+b zpq!VfRbyUQqNt>V4JutogAPyNh*qc3b65x(c$7Z^|Ko5Aw}^-%CdLX+&~^3V^j3Vj zZp8AeGdUSQ40VirXU90KKL0K5LcE!W{GIZ`>$p?KEF?q6T^`kBqhg%F6ZHjt2sDdZ$5}Z>$K46Jk<&4{;ut^h*8IFTg$ni)x)81kdwL}!$j7YI}<(p(_=vG9rx%r%5%<5+u z_C?TXrxFNw)|-1!IPwrfy`PSIQv*{)lyBcQQmcP1TZ=hN&$nYTWon*0AwH;Yg~vL@ zqA}J-jZE++(k*mUR9%u0WnGbECjSV}&g@=AwcCSEDHqVD=|}TXRmcw-h|mNHf2jnD z#k4YI#UG7ZK}MLQT-^8_4(NNCy2K1HF-mkrm|@uuQATOfVTgQ1uy)k2E;bpb7D9jD z0K339hE=$6t=0EZ(e^?`S3W$zSOj;BJ&_&a4n{8fBP^cdpNa+CKQ5n0M^R8pGSDF< zdiEaYF5jBn^y=!~__b zga(*`9l*4Ba#kZ!QaK3r!cxgL8qoCn!_$r=hwwF>$cpRG|VWnX-tx2JZo$smKT&S~%@4daeF;y*?dwmcS;`eBNMJw5P6$ z-YfAI1tlJhQZBGmzdgjTL!v6ur$rfu=*Q=b!1#qFA*m80F1%7zra?*8J7beIHUuS2 z*1rR?IrBVmagZ#w{fq>G)E`7xTbpRe$l}+#PIdZ*&mx+o;Y-9|F7#lCtg5^sT3Uz$ zT#5^Qv#;<;MMD^g!N{c$!r44qKuB~jl9IcmBto72`Q#q{7q_0D$F7@Cs@{di`-Wws z>6u6xprt41d%tP}+H@@>tJwK?r9|(aWeas;A=KiuZ$NO$HPwg5wW(wAy%UTMv997u zUNa$vUPO1ItjCKKm5&T}&C;@btJUn*luJql1L1KWBn{#qrN(t^&^%I7KbtwT*e!5K z(VEi6-p8ek2we$rEo^_peXmP*?%~~QFE2lQaoWvIP0dG~?klX=w_YqiQW2fjzP4N! z`pZ(M?luQYrJK0ysa8M7{>f~}?FVLPA7;y5M^{f{(MtjPfFFt54DjPA3O5iryh=a1RtcA`sW>g^>lz3s)G&5h zv!}QnS@4@PVz5}5E3NsJr>V(dEkSJ^rZNySK{0(voAsbPN=9e;RbimfNGGfzAIXsD zfJkAuMWfU^qw+D1Y~OpA>99M2(d@db4>N6dH_>YYkC2kul(Fs4n$N|N^$y8ve|7sZ#AS>0#mbR+5;)sUFPwRT;zk;?7RJ;cdn4Ow9y7>Z#BK;AVJUOUlc4Tf&oju z>+$-g+s_INuByuL@!UPu1G30hg2Ujjt8COiz1ZB2p!Cx+=)d=yv**F-v+2$liUifK zKKT^wVQ8nxYR@yzmMw%ODFz?EqWG1kdAhORJ%pBL!Z$c%4BEByRZL_Bq4xk#Rq&?F z8f*$ zSYRu?WWDoa#S{5XcV{& zb=9dv#UX}Q?--ZfcPR$*bAowC3NT4{M3evoxOOPDERB)FYyu(O>+%(pbEbo@hKY#@ zP*>5a2SA<_DsCu8dM?kPvT*^ozdnLwv#^;4_02JX=_Sgmj`7*Csg9~}KU7cN?2pT+ zt+-PLGKGw|Z&2RMOQlxU21AU^cIPcJBnnhzL}W~&QU+E?&_ixB_<1g!cF{= z+V)2ZmfU&uGLQ-WzPv?(g-2WRI|2CFzS$c9TCW6gjy?ql&9u&2&gL1;GF6 z%lI$E^JeK2zs^H#ky~7-N^f0B>qz>gozY7fdy{{}*Ux~(n-9=rQ0GZV9Y}<@li_DE znbh(^lrX1{`jPH9=e-1on4TIrfgZ4$#pupXk1kDd*y`Kt3$F0MMy2G~D$;z5)7dHg zB&G*W7xjykiPEtIu>T~ZhJPYHC*EJNps%xs)ys|9bO_qsr$6mPf{klO` zy{kU8kYCsNLe%P+`(B5rq%3C1gTr`3^Ps*0*E&bNz-;%mrTJZfSdlgMtHgWvae!dc z`I~Jd375CIX~YL$jC9MK;KA~aj-8!2qItt0Ec^AeAHOh0O}k3kH%tHWEsN^&h}L~g z{4w_O;k6w--YJGHMH%#qkQp|2LA$jr*s!}I&Xu4D;UGVFYc8I(#w>D1F76HADrfB& zBb&&Vj9O=bnk;?gUxeSW)33ac4dsN64-^LKfFi3pS6!f`QX>u7LmBdfVG`Fy$eeLE~Rt#xiS)fC9bIS{Fv?h zi0XRB_3KU^$BY{;(&YyIn=Den+D@>&{Mz66^bL@n%gVnyEPR5D>fH9+0z-wX`Mk2L z<*6Y-jTMgC58a;-nYl&~Rvz}Kp>3R=NkiuSQqb}`sbQke5)*X2QasCXId&1P_n zleN_{d(B_XFgdf7BKExoVHd=(@&QYE>l{E&0gw z&bX8D519N#mm-K`sLyKTJI-PRHzc~T;>?aWwn(AL72S7Esx=ynJvyCfQT%gWhMboH z2O~1l-fZhsk33fig#NKKlB#?K5Nb-ku))*Big_}Zpe~=7k?8M{kctF|Ip>I* z&pjb;=;?={MQ-K?Il^PiRd&8AtS9`)1-&Nn-Lb8jhC5cjEYJ*hDLbJ`+TwU)Xz+h{ zeMJ*O#{?$A+X;pp1Bg(8!8U+xkT+iZ*@s`Cy+yNNrq}-HS@1!9H17vtTFGtk9>JB& z<||z0!-fiEFf{(CWB$Ya@Xy?kIdiOtR0?H52({`NA+AlfQsg4sJoU##b+H>EOb|}i zNL43g!t8C(w!7XL*rS6_tI_=W76C-GVMo35rW}VLk%bdkol4&!^z1@~szrEEFkv&` z?-hQ{J~^@FyV;CM8e-CI^!M?B`6rE_HZZw-4X#o&oxZlH(eEP9A^FBI@3@DDbYDwK z3r=fLT;iv-T23hW^gMhE8}%NyA|z^&GB8;F=9)9}ZVm(|J2yJl<9`R_#wBM4-7Vko zLnzQyAHYb^CcKrdPziVTfd+U!me-de#7}eS#hhXQtO+qr`vF@vv3(TpU znnFJ31JvnjP|P73V0Bss*IL=k-srTra{xfFCG?F`d_$$06`!h`+xr^V!e96TsUzN! z&rO=6P##Q)f#YBBvNTez27Hec?;{X_pSU*#6PC`YUuW}Rtr%jPots-2SeYh=8Q-c| z_<07SxFkT`qAdhR7d674WS|&AO*}94f|63cuQRe>Jz-6Ms;%gy0(F&tft&$xjqFUW zUKEoWnpv>A_i|{=n~FI{IvoDo!)~Amz1TFpfz9CKa&=p0l3+T+&o1s?q=jMQk*m60 zi?gH41VNx3066*Ag?yiC$af%UvamF9X4STE zsebI1MZ13xGO=a<^Bu{xIoC>+xH-$=^|!;9HK)Iwd@j>*`pqf(qxT$HCE8P?k~5^@ z&?se~nIHp6-(2`QjQmdv>t=Lujt`j^Y>UjQS7p{5+lI9Iob)7fa5fKiFZShEItx-N zcHTQRNx`p_A1q7%8QxuvE3)k}jt4v*{^fwY%NzE*hjS20Ac+sZ6M^6NjiL5Ozp!wl{|& zqtCKk5xB%NBqc?Llo+PTGGnqTp0*x`v8g^@$Q8ub{q2?Gct$%8QIbVT%=Ry^j~>KZ zMvln~CHMBOz-qm}Lol&!@&Te3pP(Ur?^V7Z<3)9`OT5X-W7jp)?{{;&xp3~hp0m!) zQWL`G8E1#ly@1`QqAyWrz1*20?5EIRZ%jc!v9>u7ldqJ6%Exb&g={FPDFEbQ7I6Nd z&eSh&*fQS!AS6{Vh0%(&{KOV$uw3y1b^Wn-fxdcZ*lXNA1?}e;>z-fu0-W+K6&F43dl@|43=j@d4X4lB7rMq<=_lP{ zz9UM|($Yf3`tr{`sBt}Vts)atl5iA2Pn!)rMSFD`=MDBK6`*Pbqp_!w}$5<_lLJY zcqypqr*Z%D`7DUUl2m=pmbCe)Q^HJvutad4N)H=?3?&~!o|WctuCc<*Dlz=m)yTwb zb{UjmOf*!sTecmK&Qnnf3H6PUiDSKP}tM zFOliD??o01Yn81|()oVmcTqgtMP(ew;v7uXUNQadm}>fShJpa39{uK_{spPP?P<}l(trn(h>_VT!T zNh|A}Y<2<31|}9sYh%uocbDU1+wI-yGG3C z9TZmaD(yyyUd*aM#Ht zRpWfvD8)_U!+QcU$&@T`#cww(b#w^Vh(g_4QF*=T$G1fInN0a+1!}X!B5Wyh(M1tD z>W6$3T=oOcBD8Qp?yC6V#j{DC8xarzVy(6Ftb(5QAu5VpewQ5-lrj*8vWy80{8QUr zn=5#A?iIi9IdyNGaALekKPK^Pyc~ogTMG?QSLn<86qNL zfe4!0JfNw9?$78yr#G1waS2bGRJgPOMer-j4*kHgjl=JCep$d>yAeHDt4QapAMu2E z@Fe=0X_Cm*5dqaN@X&7{>P?xo!LsL}pQIE=+pZ!qZm9e3eGY1@^i1W{A@6e|^yZ~X zoo^I2b6yW*y`)6wy9W|w4juVO8;V7)3aw=csQ5q}Y4Nz~^+$R63jnXGI<)b(Qs>!?Cr{yivYR2k09&-&B21vL8i@WD_)#nIxvH@EsrOr8rrgKWhRrTxJ2!V_bxEZf5j!o zBUM{|kgz=;8VVmT{_qAZqPrHQJ;!Rn6?k%zbw^r6@no?%K&`8W{+5r0$iwDHHAfno zT{om5_Qv27G^5%dDUfI$ch(Wo5xF+q-~qg8NZMom>q!NRsu}t-t6d`LcoMSmJ^L1` zA{rqxb+Lw)+V?XyiF;+iG=iJb^EICHU*hiksIQ?nrfzuoErb=j_sFfI1EKpr|l&wAFfCKYQ zhvE~}7oE~5ByliKgs5lN@RJQ7HjcslkX1iZ(7?T-c@*+uMs!0oR_lv_^4*X_%F+RJqS1_IG^&8E{oY)-$jKJ5nH(pN={b{0M#nyzQ8U`Lv{2G12<=Rt(g zGOQ99cpVk^n}LNFxq86!JRemb%LUC3(eBdH(LhEG{zx`(XqRuZ3TU{i{KN9dJ@tOk zs0Du;_M1|7@1wG@ThRJt zhll9ng38x5FJYrv7+QXQeswQ3vp|52qlEGwbFLXx3uFHkWA8=M+#^@GwfJxFKJH_x zzG^9+zh6!`U9T+eP<+WwuZ{41W?j(7poC3DWe=dC? zBE}M5R1s$G4fS@BTvE}U`dBe!HscF?mKQUKG$>$kHb35Zi%@14V(0w*d7AjEsFcpE zQ5{lq+*Oh_RiFL$GQ^!Y2C3FPLnyTW0CocK-p+Jt!A2Z3Z~^7OQonsWIFzXzT7LOk z3%ErK#DBG^eJxti1omP7$av%;J0^VbVXqvNy#(6s4#jJ311B+3aT3vQrIXpuI+wIn zX(S0xv|wwdPNWXiRY+yTyCO{DnTav@i+ySM&$&pvbL{u(RYem}Od2wEjuSV^djSrU z{k!bF;`sip1v37NDP=mluJTV^rO3vG-DO+Uv+{Q$ z;g!8pH@I;`Tgdcrw9II=nt-#zs&SFMy(+(!b#Zeo|zC7Hk_t=(0sC#8uxq&CY3E0_RTpwAaN|I_>iEC(Q`Z4DBtM9i%h zt~hozE3dQk>-5`q;`*lR>x#&|zt+(1#WHqxM9*>I)q5+vwh<5kr(yguvho0)xoea_ z;sU1y8ma!*;*~sc2^nnY43;#lDA0NO;dCaTuKZ#_P1S1*MXu1?ig02||03;E$$06M zvMb1nupPnDg`u(wNl8n0fg9_^3xkK$JK)za8;~8(;}z(ct}imb1nk(gK6hGw=x|1Jn5mkY{^E2lkbX2&rd(^wzMc;f; z9l^ST)O|=U*=KuvE{Uo5Rn%O@DqD=>L%169C$Q|{Ax-fOFwdEub(V>D@{ey(W zvqfmp#`yKeJ^VjvOdZ`aDiA;i%&#VfOSV~@FHks+_Dg~B)m-$unkPXy2wmv_w|XKv zS0&SH?E}AoUr$Ql%Y#f9uJ3rMpntY$26PW&gJWkzXJ_|pa$=TSf*P3w0{`sS%WI0x zRPp)}9T@>kT>~ovVd)Um*Wg}%{ob-4RK&TVIJCavByI@MjaWR+2VbBCM>H-ko-^9U zRruIJm$A-}o!a8jm)`il1XovwYhPB1WEHE7-e5*#4^Dw{{)cl_75e;eo z)zw$rr#vKD-1zEsCEPKEPMm0*pL(XpIPm>Z!pJ2fitbWa6neYJ37JprD;GSS*fn9! z6HG6or;#M^ne^B%w83u&K~r=fcB>~{=a-u(t6F!kbnX`{&Q2=dboHX^S8Vkwhhc8Dl`2s(Yc2jIXnWs_&F-P{G?$ z56nWJ9{yaIH_Ix#Zr0ck)ySfKJ@)y2LD_ssA5nZnp@z$5FA4%fe}uUMfh}O2_o7-e zNzL&y;xR+{lOgZ&T{>7)0{z%0mYE^uXNxW!-;lg`qnEkKh)m0n9iUW?=gWv*NxiZ6l5Ua(2sheY83WSYOJ|BMNa0 zZL^DCBl??tBL`N}STmDea?$v=0^Gy}(Ie~F*GxKaPkt@Z&m70Y6$MVc|^&0wb{equTJFcEy`I6()6a2!73t{C_k}C$pjhWsdc4QLb zPtd=oyKm$b?I)|XHFv}aJ?ixxTr|w%$rES#$P@Mwfb7Vso->y~vB}btH@bj@R8>_; z?H=y!C38Ok?WA|D!w-nb7GO09-l|s~KL^Fz%uZ2r45%kVC$gb+qOHO?#rK$IrKn;LBQfBNwtO-vbNLYl*eM$rOBbYw21p$-zFmD8$I`6 zwH2JLS<3sBHPH+cbv9y#N~8av=|rdNiHS0#X)tv^zXEug@AO}@WJTF*XmQ1WvlzNc zF8C$&XTW@Eh-c8O6k&(~U%TQub{tMl4ex}-VMMFN195X|O9>sfHg2u3583^nGI95vgAJA@4K401AiqG7cIIsfaWX@mnxhlstjICy&(c_pQ9+Xz z2=*UjiUeZ=Jr|Pv^=OG(r_Gxf_nlfd5H>!&1KM;rcoKmT9#qs?jRM43wUHkrJcE)+ z#=jt#BTe~pAd2gka0jQqPgkF$^wK%)^D&{>9|NDg5_|b3u9*-V7ncQXOKQVc8XN3; zG%J10IP61i{E+;9+iSd2@tBZrXkq=eQc++j-S@6QOv6rplzgB|TCX%01SE4<{00h_ z`jAutBRa5NYzDIZmjq)};mv>fu2yIIcEAG!1ffK~;_o^eZTH*2IW{RiD9!tm%o{lfp7S5t;8?U z(Evh-37qbUc!tHf%OEhlp_bl0Y=vKmAU*;{ebz^^RrG=QFdd;ary#$T3lBO55fM!K zReNhHYU&NJ$fMqA9}g-w`uOt?lA~q5Z z@uwXDoQ^Jz(=Usta}4&Y>ABseBF+vEdb#Ng%iBu58+*hQb~A|iSBh`rY-4#=qFRGG2rie7n4Bt0Cq$XytgDs^El#m1p5>p?ykmk zs5+u4Y&FUaglliWE#P5-Jl1<@u{iCTXP*kytu;Oyuv<-X4==l6RiF&yZ)Wu+<^L^r zaTHQAZ*-S^CFr7BQkq?s(!e=;W=Zs1M=(E}lQdJcjQnjS)JNY6wrYS<8S(G-m^e7- zO8M|KV> z4X_3wAgEspx|;&+PB7?Qod%mi{?ko|6fW2QCV9mxz=sk3%6f9Bt<1>`Lww z(|eWiW|9MnMpBnbj7iB&e-2 zQL!N&48G8y;=7p1wVdoUMH07K7tan@>Fx z4aMuBPA^Ym#fmPOe_e^Mj47|W@uzu6)2NBuZ>Jk4>#y#?g>7Zi)hs^Gn~wIz$QrwJ zvbEh{pYJ4wZ{LQZL`lWd2ZLuP3jp*q>TQ4SiT?(msxIJ@IR(oT4Z&i+gO|ZU{k{H_ z0qZRc;3fm?R`3;c4Zz0(kT3TvA1ycCb`Ga|o6W(+ce(!xTyyjA%cgMFU?VEs+qXlb zpQ%&3fE0@d0y$k^@82{CjD?(^J0c(0`rUf(Ma3AE&H%eBK2ie)K@WAWfjRLMfHuuYeU}E0t z+wBiI}BgIRZb4Xk^g5F?L%cOWyVu7q>u9^M^ z(OHhZ)4ttj%5tWxm@C$iZ{p&K9-*MS&O(sfO9t@jn+Q!X4exiBFknACVdv0W$>S5S zzO3k}<&~0~T<^;1(6C=COun>r7V61N17=Yk?f1N&60nGBiqvpystbZXFx(QJ%qmzd zO(VOSAgvjkS>pr;(FY8W273Ww{51fC+3iiyDz|~IS@T7K($!=9)z3hZQjF5;(hC>c z1OYpG1y>!`+u^0uS444VPF)aThH@i==g@niPz-uc+yh;dekn(Mm&afihKz+gbQS13 zt7Hx~xLAsDnG&V z_*lLhn6J~#de_Mf?6MEQc^4NX!m*qY#sqYDMH2<~YA<6|sp@SVn#{Ld>)PF<0A!7K z)ZTBfh_E*C-m1AHRA{7hYXyjq`_H4Y&-)F493Z9-b?)>xO4c;;zwwy5exiHCT&8pJ zqg(*PsenyNQa|$lUX;i;>PS3Zswc=%LH&4e z6ayMy`&_Lwk8fNzT4tuaXqY(}Y%nAgT7p&JBVr($PFPjtLr^vdvvKw{`-K7c}&mvcQ?#Tg>;*Nj+IA zcllpaMdwnA3GunR!G$fE)Z*72Wqq zH9l^gfrkYbxP{wmtF+!vXX-H~&X-~B#hL`>ULBmkfhg62E5PPVe!Fq32G{mWD3qs> zS{YNe=P~7O+OK%|BjJHT1|Y*vYW$>SABDv~phzwTPA#bvp#C@iltWevEi zw4=9u$aufv8Kl+{5tRP|>oIm3D{xN>Y`=$FWV@xN*F*mD6x>h;-nf5o9K|#$(jQ^aq1Ne9T+>DY zs=Q|RN{F;vm-bhp?%SBUscO*Dc<(K~2p_sh@|Y}^|8J|wihT(#0Q1xxY%E+cOk%Pc zi`k>rr|@&(Xm#pJ@X)`i)miqP@2Hw~oqnr&Djb7pfKm!ZlhfRHwyYFtvg)*3!9bcO z?;oq22-lXCI{xDaytoVJFZJPJ<3?IkMMP$Z$t3&AuNHUH9M{037{7e9(`7{pHus3T zKnwnQp2lR4DqG;{Gm;)9VEvl&_n z0*$Na_eU=g`1lnIT@yWuf`W!4pz8!^QY4Y7)*PH3mvy>D>}(q=SQRM{Ka1&biVeu}rWNe})o{cbz~=YaNU1 zZ6$L`sbyzbpSSA9^h4oB3)OLwlLxl4BSFPAvl?l`&)gPWVwMv5kJZIE1~Z41?)ze} zKdEa@XO7{Glarvwt0kupoqFjKe~skCuH!$+kl%mbZkEM-k%r@Fee>r6aRXN(Q_6fv zB1Q9i7g7xU>B8GM%#(Jg3!mlJq~)T~OK9t4Ho&Qn%MFs$pn?V&WMm1DzaPaVlVo;0 z{VfV8H2Bb460}ACrZw)*3K~_Kn{IT0;GMtA!qA%$R(B(H>sKaPB;)jkMbf+xKn`X0 zCt+HTMoku}QeJqWUt8qAKiY%mV}rLfu=04DSuEqN>Irg#XF8X_4xa_kC_xc3X-J^u zH@<#szv9An76}(rzu@N6JF=^?8d@-XLr}Z+0$aHz8EK3Yr4xM09l-+xBy#_0ccyqA zdZ1@@MxdYpNDAi)DL+B`NC3+0y;}XM9|g58@M?T@;|I946vDD~efZ<|-m~;~(v3ce z0Oh}O?bU`FkEC<${hiS5W{kW&q6Dihm|_EW8x|-x>6D_6OW+3``hn<}qhxAa2lchQj(0LUw>Eq<=TV1kKwEC^Qz$}jPkBL+}s zxMC6)$Rx9)fZgODz-q;*bAZ|WBnzy=EVgSx_uyL$0+AK~S5UzY20KbqJSG7ZuP1Kq z-M={d67a~Z*52UBVaOrY*5$nK_(6<+3Z_F6vlf5F9E*KBtlljv3%WZ-{@(|^Y@wU& z6&BWV&qsY)uzkepV6l$LVWFb~7F6efH4@U&%BhRii?y5GzRXzD0=UIV!2b7aeoK(0Zr%}H6vLm}`U~dcf~twFk<>o{ zHa2Xv(s~kRER#wX5tN$lJ1~-kqsz^1Y>P3%7H5zG2e!_FSmK7pE#-}N_qG1QVjZ)Gn z;M*84oP~~qRVpmNBFsWHmz`g408uyJ4UD{cKrwlC&#CnN^Y89p%xVGJ`Dhu}CYR%0 z&>6pY^(q)Zn|r<>*C=_(`KE)$C4&1lX z$)OhIYg2F>dej*hE7m8@-%9R2Ycxys+6u7i$pQ{>I8^HPkJSEV_3@{jHF-Jz24^k6 zd$y>hg31AvMhFt-sKj8k+~@>4$#_(6Bb|WzXHlJ^#7z}fd1_^jMo)qHcI zUb4d?od#BK0MK?kT6zW6$99uP@ur}${0jP?7hgcF=+yiICra+<% zpzfH;oyB>_z8kJJKLklhs7|Ju^!W;lW>`Zj}yo)|@*jHC7vRz^N{*r@L62@OOIU>DQm_Oft^S08CO5T)w zdKsF|xc~k8ccyPvQ#6EJl7I#9(}&$kOG{n<#Hp7%Y|DYc45fp&yF1VAujx*Y7|JJm7bnnzYQ`biAq6VOH0*N_X?Dnt-c_G zh-k*&3t$TSg(~KU3pEE>!9kXTtJ_b#nUkSYt}X5Eu2mf=H?5=ffKqhPCtNJ(CM|pC z#9VX+_A!m9H3MT&q6oYLHEIjx7JnHprWIX zygxpeuS!zAFk&`cu>U`N?Re*VsQC`eR;TLALYZ2>oFZn#s4U*Ngg1gwKBu=}Pz-0y z`9xf(`&ypB1+cU)7te}W@OfX+bHGM~-gU_(ptYp(JL=|sJl=UDC_y7re0d$+xnbHe zLBH8#I=D&zI#lU+dSQUjs#JA~@%aLtF1iBW*cpi#-U0fAbc%IfmA)ZKxHnz)inW&W zcXG0f!L+?$k<;u3X|mK z5k6q+NN3F1@hy%gImU4*UtmANp?IYmw@sK=Cfw5GODvL7T?d9FO5(P=^_D-%T;I)f zmAg5J!d$PUd|rJG9PZJIBsBv&gVdA1dtPmlEUCym(n43eGKgOO*Af}`N)GKmhRBcJ z4h{~=s<-x1Qc{CDj~M{sl#R`^NiVzyHWm$fZyz6HI;3S!#1D>*MGC%93j*q1{aQcq*=2Lah z`!=-pF3&SeY`rXTbPh`l7iC8XBT1LRf}{U_x@c2WPdJr^Xy1ZGAV8Fu-#`n&eV41` zw9Q!{Mct8HQRtFsGyuVjYmi4W{PE+*lz|-N$A<{NhB`kIG(b78^53Z2yQib)t&<7o zq*;Km;qZJOapO4oBdxR#^C`7A8(oXT5Q_Pn$)VEwqJ=lpj~%hfAKHp^&oA@Z<@B4^ zRZ_VN?b3sk=JaWn4*2{M{182_WE!kv1Q%l17a&`UzrGczaC)2#P_rdSr=%BKd7Vjv zUC$s?{2y{S6Y(%%{7bC) z)HRagcRyD^@AZDX*&cRNs-zUwhPqnW=D-;m(p@bA2GBYWxcUOOlp$_7LkUjHV1t5# z;BogJxmJNEh;AO+cN`+IvkZCDNMA8kMK|%TX`NJ}f@6=1X0dH;k8tkxFB^E*P%}$M~ z_JR~UOfX8_ay5db4q&a@MN~Q@sQ=8f;ZFKT_=J>$UeN=gR=jx<(>_WoUO3b+DG@f4 zTlIgc(3?tu!Sz8sgAM$J`me?%A2k5kCdsqF<&ESkqy7Sk=(pFP3MIXLfF@;BRhcZ_ zK6@pSF5t!b+H3^WzNBC8D+Uv(J2XLYQVs^lxY^)yfwYZ2d^a|&aTbT};Q+fn&W!mj zBEK8V`F@m;nJ(DfPC;0&Vysy+5*;8Y*7_$63zfkW{rE;w1!IZ{6&0doLC}@L>@nMs z+5-1I?QRl)6%_uaQe7pf4M~lKTYfIN%MWTpju%+-g$|0_%+9kK97o%GflzDlnX@>= z(e!rf@3qOHq{P^>3FO`U(dpAc@;|DVZ|PtvH`FnaGg(PqI1`}9<`3!0ix};~PJm&H z?RqET!9tC+tTemo zw`a{awB`4|LXA+-;KBNV4?rbl>DJtvt%|sDHN$Y-=DpV|fdA!4<^mKG0cN+AMs(%O1{-Vg&Q`vHtaK(=dq#`f>0NAbaZU3-~cCN?;%MbH28Pz!vp^-)986j z>(~FkSVKUBTYcitzDizg7nvzEYv#jW#DR_V2RTMxHzzU+umX(bobXI(;d@PP+mOZ^^YH>*pgHj;W2rfF4(n{W|s(c$^s&OX}++ zXFs|`$7h1ABEgfnlw4=*R&;Y}twwBFqGlvBV2*dLWHD!%-MQ2MZ z{gat&*<_zSDjB=}k502+<3`S-*0EL_ml)PTG-?C)UtyTX(>#8J>@A&)3<;Un!^Lc2 zcRtueVD5f>vJT?Mxr7N3+F%{&|Hc^Ua439kfN{S5g_p3bbWI}&+b3_fC_C*ERGkrc zuR3YUHUT47XVt~824|F*X2IpWYi5Jpefud`Y8R+GHJZ{@L_`grOfs-IPcF6dbfZu8 zs|b{(uS2D}pB*#X=#&&68?+-#&tHX8G+L�b-WQ!LMZA@Vo))4k=J~KK(0bO)f3` zd2+nmGKS+OJbGS`$rV+W^7x_W4;u=L7?lwB@Njd#LieSR{5Z1*;82doP~%yGN2|(z zjAtpC%i}%euF&oV*jT0=M<6S7Ty*OjyVH5D{RDsn zHdJ%oUJlj8 z-@je*f)D;+2HK@oPD=6i|Ksbe!?Fs$E@8SuP$Z;8K?ccRbmr%RQRxgms(gCKwavN!^q<(zk>R`=?b!3u--1~qlM^_xH( z5{DmsBnaHThcOLH+^^z~XIXbxBZ=l-L~iJ!|NjL#?E~o4ca~(6vW1+cgT0$l_8yZL zm71!1^im=^?Xppr!fi5{tI%FY9jbw zkW<>b&V}p>WFP#=ZUawW5R77KIn&cshVaR$zpe(OPmu%;qgMSIHTb>4b;1-ajt|+t zT6q!U7m1_chw*x@7`OoJ)TlC>PEJ`MbXN-ZdP}v4E0O8X_$_& zV9NTs{b$X$5#HveQ!J=*CY`S1e;hvFuySzfNs!8SY6IsO{e|N9ivP}2La3UVB(|Z) zVTVdKQtTWHhR@Z1L#9w}deGeb{2COePeQ;Lp{L$-_3OdllKq4m*B_J9Z_ymog^OzL zjnv|QT`6MuEDcx$X6E61;ysicuUt5e|MuSw#}dz?6)-*k7Ftt2`U9mrwt816$>9Lc za3MX6!b^5njy=g#4SZ+hQbUA4i{0mO#g+xPeDz{CpuuX8d zU)%E8chRrHpb5hKHJP1e#ShU=nq=N5jL93>@tf@QD^#3wdN(Q&()h0{=!>Ms$0VRc z?4`f;+52-N>7R8#x?YdQdPZub565Lm_p>;=KyTiQ#XlZsMq%C8-V5-Ic`H^T&HeS@ z89xNFQ}%kIo5dL^ncyLUAyM;2ms4-qhZeBk@eF3NG%gSTGn)V_*?+9=|2&9z#rAd4 zwJC?U7Qc?V7oG%T$B7e%7 z6cf0(wEqvhsxu^Mn^p0@;AbJK16Hg3GFa7rZT#t}I)sGQFKc6Z*@g5<&L4r%qP)ZJ z-V;>^w*BN}>L~{@h&%Fl_Tl?ODlvVEa0bk37e(qJg2aU;u{lkac>8%J!sly<;a1z` zS>w#tnENVpbk8^*6+{0$RrV}l;}=Bwa>D$rTF|oU--zRc>5@afN45th?@52YSq}Kh zkTX26nAD+$n3^D;c-s-3C)9>LBc^tT!w_@xpa_B22IJ64xFcFscTl@Pa2EhQeW@I4 zF_S7N^l$Kq)SkQ2A7L@>ret<2x_|gG6L0$b{_)y=2N-uEY=ch>!!^e7`%X=|qT)f{ zEo@YmMEwM?>mz3-9<_h8ovQ}D?fi6 za}|yA_}m`$fe@95?xvJw>0~kdh!Y^|Kq2Wd_cBFg@iR;$5~ZZ(=QmBUgj6(6`<29^ zx;EzJ9~p9?1>=t9=kd(=_WfIZ%!Nqu>E2hsF1sCek!s3*0&{Zh-%UtnMv>{`V9UVlwlNrWjDi3MN6( zUFV^?474*JG3|RYF3B3vK6uBU9&}*Rv}=V^i|Q>dK;7Suh_|`T&dh}K({2>%rCH(C zrm5wQ`LiK;68503ocZd#85ce!NQwCIp2VN#QQBDZXsw(2>s#*XckID@wYK5k+8H3C z-CJn587X?AlioiMUMgkJ>ORM#(7zt0uSb6Pr_~*Mxy*x z*2@7br8~AdVzLv0&_ogxx70ub^ABDRbB57VC811BQH1*-Xfl4;xaKNOn=Wjxx%u>} zkx&W>%W(%ay8St{!iy+<1Q7U%P*mZuYT&gpnrHt%!lEY?c?_@(Xsb>i*F28#j>LnZ zOGcH{;6Pl`Y}cPl|DLXNs8XoLUWAi-niYa$dLwv;!#olgA>W1zPSYK9-o1_0@Usfu zE*|{TC%slS7u5BMYVY;O-c0YlKtQ(H7iPb>uc3r7nXk1)=qS41-+2cOUGW)Iev+1! zh1-}2WadAYsuiYau%$=12#o*a_om_fYfa|0H`#;MEM!5W67;2bi@8gLtHj6@?W}=x z6P`kSCF#s*Am6@`?#>PHD=LlXRgBWt*Z*j@C^V4Fi>sXc@R5#AB88df$?oha=%XgA zfeKmNU74NnRxWwHP^5of5*rlQ4C6)2Y2bf}@na*S)Q$abN728R)*b0= z%xYq3iJGJBZzUX{I606b%6Yb8#%nm$#42WcRW#}%3qE0;O<|F=49X=j3vK3aAlie1 zl1l07Z|JWuL&8#O0x5&BeBp2mg^3?mjC7UiN+pk_%utH9iUb^Xkmx$^twwG7a{Tcw zDxMLX7cOy;O&>}JXz;&)1dGXP{}w8-9rc?W583+Hp4&#SqWy!(Z3zDdOirhW69ob$ z*Xik=rxV9J(<$zgaExMyzo1GBM88i;%g1#42W8)C_&(mz$e3f>{HcM)?~?evWqQB~ zz72QkcpU{LpSU=G0rUION_C{j%o=R*f;ehyFvFgADw27Q*bETYpm#JFvZ9aa*au_> zg7k1;{v-f8tbbD;FR&w)<_>Px8qVVg&vam`1dqs?VaQ6Ji)|a`HT4wdERb zBg*l`>kVR}Sp;IDA>y@YwZ(yw!}k{;ypgE4Bg#OuAJC`^~49lht8g zc}pqh?sT;z6%7?HieQvp$ldvajgo%#$a`e~!4M;$3xL{{FXLgYlMvIq4Z&nInLN`> z`t51pxoWPBtQ}K{rihens(5$(HOuC>!DMyLpGObNw%b>F=JFU(!iePKsB=6Q2v)0uc{ zOTt9sX=(#gQF=`b{YHDuN7q)rwflzB#q4SF{s=sxCdp{F`85VL{{F zC!pM4Y#gVZx-8d$;hT`7q4ISyB1p^fN$I2Y=AMA>1G!5tKOJ*#2+tLCbo{s-^SH6d z9Ah+u3H=DWC-+}_HTqxeRVyrBbH=tl8Zl3@pRFDjTXT@zx4N?8>yJrTe6$3x=4Z(C zgS6~uX+at<>YFFZ?dv5eJR2mA96<<`wd2{)#{`%AV-;$R)YcJ!D(TY5eEhBP&wAwx?Tp79^Ym8Y@t!0U)lGCyn6)fhh>GYM?dJ z>~Dq-w|XuyQIf8iI(dXRL%?MDUEMQ`jP?A&TjN;aS6R8WvMI*DpgHAK+xr*}l;9wYy9I>j){d7?Ognp@2{KQ-WRNBPpZcqw z9Mi4;a@<-)DfJ+XB9`{JpZS%weHMmy~={KuL@Q~Df}G6=Gqm8n!5XN!1ukczmw?ZK&O>qlvb8A z4WsZ`vT@YlMJlOYOn`f$-+eT=K?Ke@BTTDnp?>qRYYD;PPV^LX1X^FzfQjJmRbeSF1^E-qJN zGu9_o>Pv*HOYWx^7hD3B80xQcA%qh1XzM_w(O^$qu!&&i1a&0Y3otB_+q(m+ci&Ts zQkZs!a}AEt(#1L-HU8awJvnOard~f9VYn7M&gu1AaA3im`uVH6iwDd*Kg&^a?2Z+= zdOjVG-7C^cnAO8`7%BL6y|;+z7OOdrdEoOWH;;|&iHu+EZp)urH=dxOA(J!H)<5}+ z2B7bnGhH*?V=1*DUmSraPvC#NtKOCLQ&%8e9CTgWcsZ7fswX#3e|cTb@?eFVQ0Ch=}5d@uCK9(DwcSSNcMuw$?Z9fx!cMn>DRQUP@M6 z@+9u>p;k?}(|Ish>loaS_QLXM=Q8SAQ_#w4ke|c%d>`-w_Kuf$@7gJdnUKrnWfW9zRj-6eqg=uDi)A#)LXPo!0$!Z{tQns-5N2 zs^#4Ol_Dq#?>y(>*(MrdWGPb4K*sJ@e(2bDmV_Y>Yx7i9JItS`C( zSiB`7BC@jL@JH}mN=jO7)A=N8_H^q=_04T}WrbJm)orS47gVHCQ883xlSP|>q)kDY z7A-g*EsV{YOzu_t%CT?MA{D(MJtNt2QK{deMrr#pey-~}yP&g~5L{SY{}ZYA>sQ&l zBX5|lnmh(KlIK7eT22;NQI~b@4RYPn zDp^ztjbGhii;{;DX-IKXDK-waqj`&ZztFKm%QJcwyN9R?7l`ZKZ^`wuUzJojj+`X% zVTfG%6_VF+ow}^EEu;_$xs{xv!B&EjGsJsIPBxodFGuUya{E@=B3|y7A9fHS7vO{` zNrUvZ$?BfuGcja7FNXx-X=Ju;?nlt>GQD0#RtMgg%je2dqeN>k>hlF8Fbg=)Hr+5a zgp2_9HbP~oyn)feoKvUe0s*&6<8sN0yXk|6lkrtLjjlJ<$E#3t;wL>djVIW@MEXUS z^~hne|Lmt=IDd8ryRVC&aDftQ4>>>tNyCPf(%rCB-0OUeV#!+|zzz zP^aZ}dtu7ac%R7Yg_MR=r zDBeqT=tbnSerof^;v{4><*Q$(CvVeBnMV?*&0^6n_bql0YU2h;HlcPbB1JYGwcPTd zs+1~mq%Uf=zw=(Cw_1o?(8L{(3H6Ou)Lcv&&l`|!-3e91tGzj1gnJ1>l>r5`Y5G0- zY;~3j$qJPdmEsZW;l9YV3)|^~bw69V;Q>Rn<-66XIN^F4zu7wO6uLd{On4AhED^QEyclC+(ZZ`cw;tog2DX&4*pvqy?OtKxY zT5QD*ut)kl{W3?ouIgf`ee&6}$GI6o8|236U%@!O5PyXn|MU!@IcDw}FvJ8!nDTn7 zA|11NprLu1nvPPUeAe{cyThiPj|jv6?%;)gqdO{mMh)+K_9Js!e0tsJ0i~7g=d`TW z+o`22W9F){4<9T`r10H;r1s;sFRJ1xV0fcUF>T!HcV0p-06@Puf(xr*d+-2@Q1s{q zqf#Og2S-J=n+{#&C&>yD5oY5j$FBt3xUp`wwh^SUYuP63F0|}uf5A0)tRZ@@l1{#^ znpet7bX17;(0!k8$lP~jXf{X6(2wC&F1SZ1B#BBiLgNJUEtyjc@d7Zzf#!eQ=yi4ucG&aUjBBU)+#P2)@sRV z%M6;dlymyYDMJif+*d*S@@3k{$9Df6104&;kzSn*kJ*A7RtbNj{Aq5j;N1M`Xnx$G zeu6RD*_#h5YU2j7A=xVC7O743H+z(Z7l$}B?%a*Zd{D@?viX@5dVd>n2?Zr3DM%mq zy=asri9Q(=tdP-+)1u{2f|F41J=0Q^ouyL%Dp(BOif{{I%$&+d4{U0#A1n;W0)JGe z9>5biQQzl_D3>^nn6(b^EcuNN1=JWR?PS?AQpE)JE|@VvwXhf${QmOgORJ3PyV+v~ zo0H?Ccc|+;?(Ld+B}o<3HVDj_TJ{%{O&4J>yZ6fx>rOmSJ0iCYkYgvUMJ1n;%fQaC zlCV@|dr&+@;9w#Sy@fMV46htiy)ymrY=@Agkp*mP5IsIZdj*imN6)j@j^eCDy7F1t zn8Noz$Zv%!Ue}^?*ThTxnQQSlM$%$ib_6WCZTi$RhBdS(vr49Y;v8UBrug^0d2#L7f z@wiW0sC@Du+J`Ly5wkYPTxltdk_8xZNf=u!k~}-ukIG)2SH{0BJi6^3*KWDyChZIc$`zgLaHCv=T~oOIWOAg^khE`q zXQ1nkOHYqTOe6y@n?hYh4Nqynp>}qH3lpq!@p+#1cl0u7#iCI}$F?OtqV@-n}g#oQw9f~qC7$&BslU>u(+HBKI z5wsycY3Uj_Br6nE*wq3JZ;laHzZVkDU9$3X?vN?A@x0?XmUg%9LPiF@4$c?YpohaAN8Xc37NM~ z9V2P6$2-ggiq34vEyUpW6QoxfMm(Ub4`Vqx?_E-Lc(5Jy{FzLsxO(~VBN-Dm&8E$y^bWFx zY#jYGGBT2eQ)`J+ceV#YRoHSNgNxmZsw`v7_oO=vkX@Iv#o6hyP3WEZ%=M@Wre&u*N}6&s-^NO)7{E_I5GH2@JMAb zWTY8+Q0tq;Jw80(7Jf4L0uzm|H(6*J70U*1rb;!qr`Ce+R@Vf3B`dd(jy|@)05$Dm ztJfV6^mT%Wd(!g>IYz9>Cs4%m{dr8JhJDL@(p;$UJ8sda!Hw^0VH-5~kFN9MR=&53 zt4NPy^ZOp)OPmVXRzF~3biFfODIr$}ud$tPfzK!U#gw+W$n3+?dn`-8V>0|NtB zLCWwIqEjYKic>-09ggsEJ>Qe%^st^BTfxp?sQlIFqZ_8UjG zx3J(-t;Z!{OHreJuFN0m4JP_^+O0H2xFs9H%!^7XaMu^M49bCDq{pC**j#9+>N7rU zMKZntiQxH-MsD9{qi`(K|x$!hLWRRUKIa))y zjHg&FU#y!DPdWpGWYu^$@FOGlm@lrdUkaDRTf!$T_$yD^Kdjf;qge30D7&wRxzHNj z^gfLA#sWi$&JL~J^9>t`10{c}6z3cOigY$V%MitAb%>DK8&FG}m24@VKy{lVI<#zlw{Yt-O`i>2;THz6y3>f@WzA z=94`04perKO9>+4=MVro=(CF*cT&HLTz~{w`{HwF=V;N>`!-9>4N?)4b4d&ma}AF7 z^OcfJup$D?lHGm}@{bm(-J=xuri8P%69n^^A@MYyY-LaUl$j2rxBGg zXYwJd_iBaK_RU9f1|m-iL{M}7zRfX@w0^ouFZi}a@uW52f$=z|#1}E+9%9;7LE!w>6EzU(e1dx{jwv}gEdx@A1#KdNss~q0upyKcq|A`elADe zX49h;67p~tiF@KpAC{jhprKN!T&XN~{Hz7^K7|B~*)a_Ar0Z1r!RYr}Ws=$DZO6BE zqg+bRCDSm!#l1_(a%A&i_iVXIE~wrS?SA}vcc78_gp{+YNUNTgNA!=^hmtr^1&tc~ zl}(%YE7xnJiJ2001U=PthWdXfBc{uyc2_0LU=iOmn93g2&j`NIWL z?`SkBF)=ZOL>PmBGSR|UBTs>(H8lPfZSQ$D0hLf;mYDq5%KCbcL6gfW;8e_2X?1mV zS$mL2_#WItlGmPx6h=*DEc&~;dWI*4j|bV3Ga(l~7e}l3vm43R;f)uLv;B>omaES! zHisQuOn&3HolybD$gVtXA=S$FzCY~HjVGc>fobd$R6=n1)g0eC*IB#%|X-r-hNd@KA(rg7N2a^WE&7RkDH6kO|Ce2X5|)u zdJmgYCpYiM(A}FS!OpmEEq;**Y*l}5#IO_lGnoJ{vfEQ1@raw+-&nupmE743WFoZS z5N3LS9e_x`1;GN@)Rp45Du8lLp<#Kmj+}YX+4ok8Z&SkzZ)xEpQh~bn+cZ;@-QAi! zZ%etp{+sEDD_W|o_TQa&4_1$c#`6-y9jz(j#;2z3q{x3w>f`*Owk8w8<% z03;m3o7~*pv!OuSTxt;x!Xrlk_g+UadLnW`a`Jbs(-ogr=fl890N+J7fa2TRRpqzAOLNL4Qe`)X9x zy({!;6wG7d(O#N8M;?=<@d|*#ZZ%HZP%TFaObAXlGJG_$bQ;3Rd1R#9W4>eoiCFyE z-X8;h;BZ7LZ>}SZ>(p}%dAFekPsRLC`{>h)x1W*A$@G59H&2*6)Tc_~+AIH7f%49K z1SKcMLg~*NO%w`of``-?r&|1r8!Ndaev5plGs1-RwhxC5@+W(HUt( z0`r`H0UaGa!0Wty^pJ_5(jzS;Eq#}ZiwM}s*2bkzrn1giCU{Y1EzwTuDwS-YU%Ri8 zDgo5#YM=SMHJR|G8V*H`^ZLl_0ZM@(` z2z5bM6A=NWej$UtT>Xp2;WmO1C$td<&K0#Dq^Br(nS8z$4_7w6t{`qO3{4D10j#Ke z40reTG*@+Al2TEXy|7k>$)Mc#153o*0v$+2iKz5N*5D`iI>t68^odN&QFC+iMATg( z5)sb1+uo==`~jxRlaC$P+9>SeI4DlEk4q!2tCIsx;^wM)ti2(Seyzhdp&H9A&a7g%PA+D~R@59&M<(mI2d04UM;vM$N zjY2EI_3E3{MA^%T-d`7 zl?~f-cxeK2M0rj&4pFRTHwb_%EkjdZQ~slEo1>gnGEM04!bKT4z_G+1rQ3*1XTM0i zotBIJrl8oS%uL$2mZ6zA7VU$Hr>g(1731kLmC+nA`)p{{@;k48`R0pH^+=*RBk zRR~k6^1D%(r}lq1TGdPcV!6`8ZELGGAwTX#jV!U2rDv?cUY#2O?O3plsKje0CynDs zM3_6M5$G5g(BWiO4*=QfgzCFMXBsFHv7%xPPdmWRzFBMtS&+~a4@=*5%N;N1w`Z!i7*n*MQ` zesJLmH@EZYnk9Yf0$I@bH1W#HD~kQe>nDMR#}x6_!SZa3S+68aTdTWGhEOpRESVbD z=#-4^EE{L`IfMs_$~XA7lfgpK^4#ffWXX}5$@lN`RTN;ovG!A$af#KA@`aMr}9=cJ-& zVYRZEU&a)}0);^gTXjd$ zL_$|~Jm?C86s@P7l^IzHity zW4oAc&8NC5*$Rag&a5spje~F1ob+8tvOZJXqXwcT=WiR&P~Hbw`au)OFkS?eu9+jW z5Nw9cWRTe14i){&N`n|agr4Emt5?PNsPxQiG2`4HJ%2I`EWXBw{zQzmwW72BV1cDH zaOf~U%V55}HgUua#gp4}Wn=xFIt$NDfi}$&UHYe6V>K&krkH51lf>*3+xP;>jD8^c zr>m?-VgeQ!7U7Vq59~CDU@ekz@jQf8McoR+d&ZH-$X3vQ=a9)NrDaFjP4oOF`OrpI zzZK~&Rd@&+W3gn#ud`^YnLPDfbGF&fSzQ1Tn=WZq-DLkzrmijvgpX97?<{IAj5U3$ zXuJAcH7u60aRuE_wrnKzy0;EpSaf1cL{}60%1)WZt9y4fZL1u^{ZxM(%Q?&1znqP=B5$8_(HRC684krrc%V!(?Zhom5rA zra=f|A2JL)GGADw)QA`!4^V<6o45j+r-{3@KDb*XEjh(Nq+11n=d25Fkbl>uOAqXo z3W=QbVii_Wy`)Zw&R68Qz*WSx0Muxuy%uKxQ` zX&HH;#*YWwedW?7=WMzzxpIyB3CXP*LbxsYUr@hlS^24bF&y_|Uy|_mgMH%@*SI6d zyQBBe4;;%;JD+$?PW*ivTwcJ#T%S5uBU)xSlEfhQ2N){^&iT`>|MsUwwR z{{dJ;IqJIvxP7O@tyXT58Z+GY?~uJa7@PxODQKC4z_n1Ti~+YhWQ5D zE+U%#a6QHT^XMoMi&klS)ZNDp-@afgr3uwYMJ~>L7>iq#3Un1^C;xpV7}nLq0YOB< zNfMXECx!Z6EG{yaem(W2fNE1iw^YP@lpIYEP=QLoj-gV4&Rr!tNh|-rN(}2E6*ihH zD&=X(JJ`lDhth=K!ECBA=+4Rb?NV{#5e?dmQH7ys&64((%@H^GmE zzPN~KzVmR{X-3p(SBtk=U-JS|ram&sllku`=qZ+$_`bI0FXnY_ywdL1fRYQF=|@|>6oLnc-eqcgEH9a`r(eQAWI&Qpn+qs8kvQu`e3D*P0s zO~2Vhebd?GcL>%ocFpDV!(%>&Z`y2}jUN}CGGZ~Gw266TXHL&D_!9pNKEBe%E zh-CAI9zjb=!MDi%Dz5A0&Y->SKto%x)z5LLbB)=iEWRHE#xFJX0LDK)M`q7^=p_h? zH)(IVjRqn3N0^fw!*QPT`bQtLW#dy{dIsn9;fzS=t}?+=WH&+57_RxlMgygF;Z}qZ zp~0gdHRFlAc=4d><1axa%d2hgrpRg&k|iquHjs_C_8UE3%fAr$TeIazOm;TSTyzg~ zx0SlYSaBRi)CP@CTuN!N15V(H+Fku}9I&;h9loTbB!n8xZn4oEtF&;%=Cm}T(D-wC zS4sQkpH6M#TclYYi@(Umxo!yO_CdI@x$Z}!aod`lwArm+5@jo^RQuquThcdUhRLsdiq-@H(4})y6F+R$b6BI3S;@# zk|v{$!S5+SUVKKI$5YZPE5s`!#Iz?dTVfg860xg#p>`GRF+!4XBX45?8s{tV>Z0$E zG&9S^iTNHgy{SHgB(LtSIGBr2g~R#spgX6A3NNZ~LeNF@>=b^+&$qk$w+LCLAjHN8z+YFZrI$h{jkla)UXN>*q`zI?iA z=B1_{zSSD~9^R&7OcXnF#ceFO{J!elf@<9u=+;QkAp2P^?_`LHlk)}w!H11Ae~O;X zsFe~RvTQw%qqB{SgdVPW91JCA_ z$Vgd`#{XVe2A4RX;7imPi~JEq?djQB7PJ9YDW9N~&xV3i9pBGgsWBToR@gK61Rl_kE#%-<-FmngRuvt?B9D}`gNcX2>gupwVj+^dLL`extqwCM~jU5wv`wE<|3QaJieV3P)1XNyR zyjB|JwNP~%`)Xj}MfK!wItkejPNU?AZdApxfXc(vGK`sQnu(00#o%!Itc~-Cx7^={O~kNK>f?PevuIUoW=2&GLWT`r?9qQVe5ZFrxlPgSvh3fW zNnEvl$p}|@fs!*=kfUMjbnuO>(*?m~SpQ=(GMf`v2xO_cb9pmXN{u-epDN{fc9O#BTM4}Z|AoK)%aCa?#VsV4ZG zNWi;d)Qz5SlYMQH?YcAhH=o`n@Ag9nhr14L|wX@BssDE@0I+-2~7C^E5bT(}a|l>kKu2i)gT4O*uQ zhqkQ9WVw(`O$mX$>-}7d{I#z>K3_^7R{_CyQapvr!Rvt?C&JQUb(39c7oC80;%&~v z?zBX-o;0vHR-}SW2@`5isS@e3!2%{30IBu%D&|T#rY&-vPuJWp>pJt^3P6OJlh}os6$EHZA zF3IIW0$UER6b2Dn6b4ecA%klw;w#9*f<0a>glS8s@&aZ3((yJwarMK6?BWXH^q<7x zSIW$tvs}V90#27V#1{V)0izvNdDkuPjAiQS=JAR0%BHHGvD02*FMC=OjomvDyp1W- zm0L5JUkIZAy%Ast!~iy%__(fy?SIQGh#Z9nKeo6F5;l)8paE+ekn`BZm>%fzFLE*w zrIjPrFBl!{!nCS3zgyAX^Z0=G5mmCWh{!B#3VeJXhIn`yDO`gep z=vj9u(z@uaa%sdt-+mLJrO5&<&Bx&(KGF<|iYYQ(+pOsstbZqy$;y=UxR<$qwX<|g z_Iy)iEgRbfFQVzl{sv%osI-zvF-|DGK&Wc3inc+MBzgExicrI2miDCwoH3FWES*|< z7Fp9rw^o+L;VZP@>-*BM>lLF`VQ0j~Sv-aOkzdyt@qJ1$gd-D^6y3pQN^Co(yuBw9 z`DFsH^W;3PS;KguMdeLwyPQo^Gk(V2C7y-JunsI(>-$g(ze_-tpM8|3o^^M`iGIYG zc4htPe;O7oD=1IhV9I&4(p+(OspTTVb~shgAEF5ELYod!UTN^&4BHd1MTX}{JiQjn zW#13-6)z(Bm!ROtVKzThue;$?L=sivV6CG>mcN%0w5MHKIAZ;O0fF%$^z_%IEub^!rlg|M^py&PFKl4j%7%O0PS795J?Iz=Smv0MG0#@I?6RO5 zW7p0!bq+LS5d9NK)!xz79%AHMJ)eJ3;)<3Wj9RZH2e5_- zD1w-ElDW+l(rvHLcQ~Bi98YQ=!jv=V#iuzG*j8>6an)%WAFVB6<-Rb(lqs`5Ep9JV z#&6U2i=G5+RT~cF`6JwLyBDh0Pn(&M-_u}H5TbgPBuo(fYoTL&LBfXVb$-+V?vKCm zqsGR@4#pHYiWD6j96~5d*o=MAjE7PMPa*O4)zPL}^I`WLw~aJcg+`;&?yk-G#1=tH z@t=_r?vT`iLJm~93|q;?dZV&;pkZFbD9fPXCn)*JUrNj@>-_`d(%G>n0C%Ixql^S+p%@&@wpc6T{jGtP9-WAz0+b!ji^|WBxHpz;cDXLOU3VX&} zb^)lEq*mCZ#N|o*zbnejv#cMY9X!zfAKh>J)#U}krez>i@Tmc~@QVUbSNfK54QUGs zeT2a+!o8%UzrO@6a^{>%0%fGaNc`C3FBB&Xi!`@6cJ)LMMes7Ak)N7=z`k>TiI5CF zOTyr9mopM~udH6&O`5S~Zk(2UMwG8Uqr~j`3ojd0x7j9rG;qV2`7v3!o}IT3vq>9+G7U3b)9b2E0X#6HzgX}KZ2uG|Vb#2K zRcu?GS!gKUKAZEeNe$ljLzBHI->aINX1{k26;u*W!Fkha#2Qp=I1zj{GrrJp8^f^b zwx@Rad{wu}7H*W?LcJf5X{4GZTGCKce*}cub6#QO2#7jUm>R&s$M1rFeh1 zZZZ4&Ge2|&01|79uu$cR?TCFx0ktpt-9pVQrS!V~x)JMRL${(LbfCGzF(O#Ur*2E9 z0ftG0^pLt3UFL`6e+thYyd1tMPgby*k@~M{I|fZI=2JOvIEZ+>p$u_*0gKzOFQ5-8 zcC@G*MUtPwm6z2ne1Z&=W`B)O$dNmCH&F33b}u&8l7+f zV13?Mv=d~jK3rq8rr=hth$V^gi{@;W;*;hi4>we0J#9pcHHo?2@reU?D+7$SwxdPj zegoCJUM>U@mV{+5qW?NMFgL6r4$Z3i1SZ>fDU?h|v;u62Wm-)BGs3k1T)ec=B}m!z zpCGHNz|2u#T^&C(Nt#NswwI;s60bo%+%dY?(-S?=ODXJ9`aA{7ON19L@UT>{kPxm+ zp{JH*!>zyh54*Y@{yGZveH?_uDAZ)F&yH!2pOooejSu`{J_3dws(FdwOk&m}3P);z zdMU+GD$?hT!~A5H+?19NZygltaDN3coz#cN>gq8#ZxQAo>K+@n$jE~86chfLXEwk2 z?Eqh$evWF)%%d2-iE>-PdoQQjw^4H63b`H7T1}NEv4qLOqM00qXF!gU3uc(k(Qnd! z@c^!_qM{NcZJ*asUNw4Z@|eseHH6Q`@%6{f%yrEJn*w8vL+9Z?weX5j%{#CLO_FPI z2O5sY_{Fwa=SXi{aV>8AH?)#9;vdFuh4z30khGw-wl)g$m|DRt*>E7%0>F%l^z3ks zI0Zk-j3QkLTw2=p`r>xag=6ZU18x+wL*seys5OM<3ry<}k;I99rUSGAAHDIrx_UTj zOH;JFlGwS#cTW*5Aca+O@+7&rj}8NuI5e(s;~6oy@mKG`j5nJ|lcoqpiKRi3tr&2~8B%XD0fa zylopNps6TPG_;ioC(B}YZEO*Qrl|9h^v7$QHIZ3!3X~64r*K`6`fo;xws>C4E(XV*(Z(IBiU&khl@_l3reb0W= zg|xP4l&;C=+SkDS^$GjTBkdeq?mMCc27`tUE>3uZrb{DHp2L#`+{IVaByThLRJGEqeAV7_5T43Yko#=>W=Izw;`G76zb)0J7ajd z@bexw;s1IdyW(DpG*j11S?N~V_~wg0DaoLo zu88z5!)e)`CWb#pAIq3HU5<7=-Q*P|5onPq(RF&^Pa(?f;DE6$Qp%uTxFj!@B<&-D zs58x~Z1^N%oPuMfo{Uj%@6h&s5Qe~@$#=F4U3KAvybRO=wd%OxbSKICj)xc&coQf&q3@H@QxKOQ7-B}HCnt}w>gd)LJ4zA@BBZg? z$*pl%l|=Z1>wsnh1~*1NK7U+l*g}DD`(vAN6~o{sR#&bqYVWZ;>T$cZrC}yAhqjR+ z@y%)AB?;MHePxatZcAU(blxH~hifIi)s#uUi=9XEM|w4cHvaB_3>3LC1!o_be-bXqpIvP5 zdhO1AJ3~LbBGU8y{8k+xxtNob0Ty>eamDNCe}=eb6eIz$N^ZO~Xvq<1 zZ<~+_JUtN2xZ2@2@=dQX(j+kD5!i;6_HCj$nXvDT&^@8Mlbj_gebnS}V(QIjk6w8D=+%Sw^`U@qaDk1yjU= zfJuD@bdiZOo^*4XRu)UBN-g-Qr<%Co>^8%%4yYwAuOwolCF0_h>LJzw*jPX0m^hDG zM@|C$Mn5Wk@3*nTD)mD7pJzV_Ei?~zurHFoFgPMxJf!T z>rRNUeRx8VfHf$ ziS*jJe)|nEs&~^1_08iDcU5DJOSN^DBlinN2eSr{P>9<#4!5N8d8tg@KAYwqUi-IB zNRThG>z^`%VlaK3yk8E{u%+n|Nt`{%W=q0RZXOHA3MB#Bdu zm4f?zFKc{ju6qe0K$AE^-EJD@do7H8^w+lo@jKP-f_h6gPPccb62|iM_gtZ00>HWx z1aKtG8tq`95gZT@pueZCu8tV4LRf6WaWF|}P-jPV!H6pxRhoz<+ONXqPqka^`+$X0 zp+`xA0!B7e^G61Ma?uR-@i$##wmK6D)T0ShqXoB^H4YaDjhNNPXV0@7T+x7KYjEL* zScw8KRAro(lvIkfq7rgKsOTSqyVL>n27ob2P7>hb;&wtBfYO{8$Zb#IueqLoMhgfE z`quQb^)aCNpI(kLk{HLi;KBXS>w~>5-)ww@lm1Y1YKg7;i^i;Oa8Xu8Rw>D(ji`?QO&euPF>c)H^YK-OZ!~ zN1QTV^WO( zg5%eGEozb1YLVEwk?hQW?z{oEg-Gox-F=&h|iUHQ4VxZF0g-f7Stfq{9+*EcZH^9jV^c;wtt z%F`E{IgwF`iG?z)^bf{7){`taEruu&ULlF19<>x*F~-}FSn=^zs)P6ob*<2{)M#@ER0ESEQi-@l^cl1 zAVq;KLJxT?Ip7uvgR230+VoHN&6xE;aH-MM(<66W1@ujWq0nrtv$d9FMQQC>%}p?M zZ`kSQutgYZ;86+P0_KI;(t0G*4;YDhdIw_e%84oq)l!{Ll4TymK7SA2C7(Nd1fzX<{@XhZ0OKG zOq7(IsA0Y^*v!)dEG6)#Aqh4VNav;F*5mt^?lHTx9NeMz5(9>o`t;Cg>%J7|jpU-!mGW31nI1%nW}xM~y0#cj z=e3D-0a2{hqjVz*nCN zwF|uQM^mf0uo(!HQB7s!EX_sv<=~vrz&HQUU@Z zAVCO`BAtXJ=h<^+{qMc&&WH2i>@T<$#boE*@B2Kz=U2Qx8kF{GK~(uUFzHlLQRxmm zkercYIh4|SvtveNjj$?Cb-KAncj(5Qye3H^tvkKET(RFprIC>!F|Y{j)6o!qk5`M15U z0@8-b->go%z+mD1+uIM^Ug~Uv4Ll=5617`bS2tjvZi~hfY~=u5rhcGVIf=wCLMD?v zBBKjsIxVBSEX!482O^9-vZlsbt5o+heZ|k+w%RAyBJ|3ohso4h`f|zR(J;6{*!pWx zbL5yw(!)$zfrtX&YMzO)0=W`tZ(hHsR~GVDA9fW`-XAiHu8GR293Aodwd4Bg`Zs+W z9)7tLcwR&)otp_WaAAdIe7&Gq*4prDKknn%MWoGn>rspvZRJzQqZt>=CikV^@AhXg3cP%xrC= z#B|nemC%kr+;M3zwaC|!x9Qt|Yj`~XEnlQBeFbI<5^n_4U9k%49@*q4Cw#wYzE?}e zR)TJ-{7Rll)HUo6>1c6rPbYuF&fwNOJJ&9*i4Ytw$%QigC|@73)s8T!AeWOZGO1yi zinOw#6s^x!G(Z0oe53Rl6P1X(4k54E;kO>qR zKT0N*1Ec5+)v9o9RUO&cdXDso|@6QLt_9ADgV>pZ3|9{sBVfOXIX?et(rBi=O?|Cn)rG#diHkDJ}NTS1)$?pB%Nb@oNV*Q1OpY zleb?0uN7`TjQ@UOxM_QpTJAvg8;iu}8%-PcpXcwMIVg9nc zp2nYJbb$U@*aHr0aLqNWqxP4Fiv2c~jFX9uMM<#-EDuhj;FO2?#!2|2DG@?j9Qpk4 zJ7?n<{)TjlS0ZPx7V_N#l*2iz6&rV~`8~;s+>tUM_%&jEFyZ#3wh)b>V%Iu*4I*uRPIsM}skuY8 z$$gT00(n(;Fe754Yi!>lYJ|z$ylq?b zpLnW980P*u^vM}GXn!!aBk%3dQ4RSLTk;NwNy*I_P*y+WdNRc#axUqV)%}G`{_6kU z>V)|C_>!`aap1!|NdU`7Abvy9wg$lFKlbh0qNAfDoZinD1yX$Ex(sZFzpuD^>yd*3 z&gI<#2ulerM+P&FW$|VQvIYNM9sev6~DAd>eVN!aL&CS85h5y?(y`>D>Qm;Cl{#)oatbjD(Rv>GB+Ya~bEhS65{x;~^5m zm!S789Yh3&8Sm4BK5$?8{M2DK{^yIU)b8uk_poF=`CG1BwOi9v1u#%VE=TpS|Nh6n|E0fg-~V%yiNdSHceY({cYgqA=6HYQrKYWA zzC-(gEet9SKG)!11|YDadn6KxVZb{=!K~iL#|Ko(!f+5en6;-bXh0DLp46M&+}vm< zIs@E-f89fiFQr@-g4!2YLP*>=Edb8;MyPoo1I6}f8K74Rt6Fl5%LuBrs6iTv9U?5d zqp|m@J;H4aB{sP%Vhq0MTm zE>6JyhnRr8p=_vgwZUqBzGld`ZdCx`5pzT!VF%<&Kl-LYeR2!Z=g(GnIo`tKM`Dfk zzwLYF-sfU4yV_-HYN`*VyWK#$Hd~=LB!sK*iU(9iWhm2*0iKbf!vaD8v>EEbD=)mc z-0oPr*M1CQ+u)$f_{hj}kZ7h->p&zp>FnZy15cxlPsyrA%Z=i`6TlVq+orv8p_nbE zf_d21(P4E02(p6?ZiH;a&7D#QYwgcqwe0s`v5?0Iew zf0bIV-wKcB=B-rj6W*P!TUk_uriE2TwsELgT|^wSQ4HkQ-_y-8n}3Q0k~a* zAKM}?#|VV_u5EEMydiY0hRoRm^RaaY-Rf$JLwQ_EL_|dJz{?p0(ppABdkgSl%_^YH zu=cLU&*xCuxQFpnDwTqi#ym*I{=weFg=r71s;Y{i==}P^7jTI^Npt19Q2X%VZz&`a zX&OG96JDg9vxv}S=lH_Hg5Hqapc{`$?S%I8=Ab{I#rd^0KLSCe(X+a`N+zjXf^mzD z+!(99I40fLryjID?<+`hmXmw;?)8Iuc2D)Eh%}vASYC{p+FIO99or{${SKYbguc+$ zL?s8deX%pXiTx^@d+9c2BiM##gj|%+z)CyZmX`azqeDE}WP+Ct!iPy{@w~stdecK9 zHpu@P;&B~nrN++RRM{p*CG{>qzc(2OenG{j!wK$Hm@G?{(xF3#di_^>oLD4|JAQg$ zldGX1K=^T=%;0D;I()Ak9I7DeUlpu<&7C|Wlh+?(s}n{ z3WU@313td3N?JUcS(QRv;7M%xw}q>}4-7Rt zarZ8SKRP-7T>rtNpZGX6#3;Vx)kohFyMlx#Zijo*2uqIdpW&BM#wI}?$I=K~xVl&c zkL`{GCAZ=qZmleB#|>|}ulCPrz&ROMJVas*g=&Oan($g!@z16f*qejWY>ySY*5-#K z9Cm1XVa}tB|4anF)LS^yUbyw3LFXYo#65NM$M4Yi_TjFS!RH#x<18a|V<#q21U3;% zXoM%LLAf;yi1h=|Af3!5!@Lgc9*mw&=?EC__5s}#^b&kv?^CO}Y-DJdp9b>IkL8vX zE>+luEbox1yEo2LcpJGpUbln|0Yev>H(}re#Z9*7rUpJDxQ}2u@`8Uz|AmJB?rgb- zUHl8(y22zdp!Rs*p1Sz@yc+~l*vP!-s030~aPiL9&+6qi--P#tk$N5ctGmEK>Af05 zWYeZi&`#4?oo9E)Guf6yI%`kQX<{0QYuvq>w~m6whva(--c36q`{W%tU-3H>%SEiO z5c9~-<#H!yz5@}}yOg)rbfJS0m64f=%o!1LBx~OfHDGE(PE%A=tXe`Z{lN(#46OBw z4T&VnsYYNR87_$|EMqQClRjE%KUT5U30m@s|I5PC^`_wcxj)xVyiDg9{g6uT zz;ttfUtD(yc>|Vu_J$Y@xxsSJ#Fh{R>4GdwKFNzXmz`0x2>Zs4Qw?$e4S3;Mby(h=E=Iyh_uY>Xsp+WWK{)36WG!MYl6!4J*C}*d@1euOe3N zNsyFi(iI}Nw&u9MvamgZU!OH>+^w5gg|+gQspCA1Sx1e&l^bzMAfYt7kiC7Noo%gwLbd>^#m`@(lXm?*2 zKc*sM!E<~`<{oAo3yeE~R;mH5i?ay&*O862bRO}$t+DPo@||5GBJQs_Ai=ORX#npA z(F(9WIqcwMPW?+_$BqnZb>b?x>7TCcfi3hkbqRK|{IZ{5W(>>f#e@BU#=@&`?uMFI zEHis8vsbV19NnJuyQks}XtNrb@M0OJXAT;^%FZl$2ZmwcIiQd$y>gV{(m9LyM9G z;BhXn0d+q`!jdm_l!oT2*_VO$dNO+n9W1D3|E`DuYOl`Z|klFnLz$*`a%ad-go1wCM;{{(J!7| zFeQkSpQ09rvOI!7P}VJC&`4j-{MrspiCu>`L1qOEVUx-ZerIzV~2IChu+ zxOHpu%7NoibNLM_kVQlbW%0FJuj^0xTC$K5+@`Zi-#G|cQQ9bc$dkox^=(MCqLd3p z4`Ys-(6jUIcuv6Lq}`%HFnE3XL2KMW0RBNhze@ z(IbDw1@BaBpk1cRD=0(^+VQ8=hY1+NK6HhbqH)u(w>~s<(J}T-9@Hlkz>mLp@%pGb zGFqRak=E&9^NyqF4S)9(|JtSfk6O8Z|DlGf2-$t0;t1F`Ak7IpjpQkK5e6VQ8kwBT zuK?0+>j73ad_t3Y7;_+RlxQ0vZ3J3EXuG2n(lRcOfaetAaRF!`KLQ#H2>c1kB#$5q zxT~VauKodGz2S_mT!a+!n~;#6pWiX?np1vVRdo?^6g;Z=&tfzH8x0HihHv`gGq{0F zVHFq~z#GpV5k-SFAaur$p-RaFAY<|=6KOhhcx$R4Hop&)BJut-jhg}hlDq-x1rkLp zTG2XT6&$Pyr@fXY=oVxaye53scm9(UaJ&=R{R+?zKyYHr1@{jRwHw*(eX#V|f1(bXOU$5lr$_K3Vh_P4q5n{`#=f zBM2(FZvSfql|josnk&P1W|x-kS65e4qNfHbpK7L>afkyc>Nogc=TZ+24*pz62zt*2 zNpL2DVhW{2_-_$Y;Rm`&%i=S775}D91^!Ste1MK|^IBl@X#MQF&F4xng21oJ%@J>D zPI?`@ssgyxJ!sMjrb>Ar4E=-y)3fapy(MWvKLI|N1@LJUWL;M%?Fyv8X z*P$J|ci#tj>@MWFTz?TkWcLEJDtB=$o-XFf)#LteA~0ja_yF^8^F|qBbJH__3SCMn(2T8 z(?VF(*^s!Vc=`Of{mQbO!yL(hF)b-xVtXHK53#Ya7d!2HnCPI@2xA^~oNC=O3+6N5 z!2+X0YXwdLS~w+Q*H|_rZ+P&a0(oBCDwthp!-AIXv4~r+Rf>8`cuoxIhM>hacmi-? zhjbr(d3^%~>oVIyc8vbz+qeBdB0mjXl-=wB-3VpD8a;h|+9e=Plusen4Cj4ovwPI3 zWp#OZ2N4?FzM;2wHf_nYS2=Pv%F3c!&MfIRB60z8+@W+L=~r0HCoG(pO z|EWDrl|J6;HxfzC8sQ1@;h=+?e0j#&dIwll-bdI}y+%D)s*^%hdytIW(%fEP)!&Ck zECcWo^}vNZCms$`a=>#@DL8r#SfB_}6Gv*>SDHkDiiEhCvt zQrnS^9xnDpEu&_5NatOv!-ayB%vKK35fZ#hFZTv8M4SCnwa~48KQPv8(zf9v@7AH~`A#;Ik69@hAeSe_F+AI$M23TDu+u8Q*n`;21K$E62 zjiP((3eQ;uj5cl|P3hnNy#pT@inOQfOQshm|^)ohZ+{l2r zMYwa5WYe|aLf9>=o62xZdy+qF$ac$B1hzj8c%_WVK!9=T;Q27@H+B;{1-*PK!RZfd zA$9<)!2nZdDsdz^QXK1~zdxl`7&TyPXRySkI5l9~D)-g-PdSmM=8_NEq7`6H2`1imlvVqfU1VqQUS!lUV7b=W7jg^DNG(=0X;XPp8(br_q_ov zr9jh8?{hnIX5(9VfXnVl3ku&gLs52eLd4|d6{rcaHkI_J0aPq<4{)*9f>b&WFjFy@ zw@JZwe9R2_vLYiaCVZ7d#KvM`!AL?)EXR?0HSDXaq2j_qSAvj9jy-r$^jg*Q zzo2wK#~E4?#b8e7r0;_|uN=uKvjXS>(zQNc4@$N^h>~44$-z<|>Ooy-zW==8@LldI z)945!;`KV0Fa{^wB$9;=+9+;r1>h+1^90Ly`I-Nd)hO zj~c8FIDz*Pqs3kGWi=(pKbotGiOqY)Y{M*kULhb8ljJfu>Y@XESRFm5YI83ySQwsI zq!Umj&qL0j)z#JWIy7N*HvD~GU~0SYS0_in8Xr4s{PkLX48LgUHyDkNkJW4wG?Nx> zc^u*qhw0g=F+dF=goTGSudb-1Smq>C7j&!W25ISfQMx_76P7uP7}iKFaXb`=s?MGt zu-Z!?RYa`+j&HQGn3Y+#@92fwLOmF#(ZSSt#H3~f?vDdAgwF4p^PWn162_mX5`WZP zB@zbc`DrSH<~ee^7x1DIHoC@=GM>58r_#)NNyn{9zZr#Lldl}4zZ@gTdcK=p_(Ci@ z)BGfVZDPnBzewgcRxxi&;-VfRS55|0b2u%`U-$SouWk%1oz8uM z6|idbw2w&3)eM1B)OMCA!s$Px;UgAH>i9#fA?CO$g5aQz0hkq#T9TsdD1yrLR^gMG zx||YKSkecLsL7Y>^+GJoIp%{?wNb<<*C?BwZtB6LLMh@nYJkp>iazo-mOm6vRl2JB zpt^P%TqqG#94Z6@j&-|n-J)8XXs#T37l!}MEtc)YiJixduj5L}W9ix$ezf7Ph0GoJ z*gwlz9ekX!BT`+(7~K#+*`^u4#^{cQ?b6q)25!b&QuqIvbkcQn-2@PmKGEzAx6id$ z%Vr8TqFm?Jwn=Cg8s44ToWFp{GVZ4u@NGx?0SDYSFC87l@u^H)XQ&8Ok1Rq~N1XAz z`~ZUGx-buqnNbzPMeb@>>0L~1s*SNaRAsEUjjAqpFDm0+C)x>pD}LpeikqS&WU)k;c9y_FtrXA<;Sr^4 zolP3II;UcHWSo6qd_JKmVV%Px(Q({QRPf3${j#Q*{OwSsTozVNaAwxFHaQe znaaJhJqLsnYJBPHknVWti)_>3Eb)$PA^zs*Xm@vOS23U7=hRfh0HxFrP5s)(`R6FIVwF?x;L>*4&{V>*XTm#9qbUX`?SUM*zvgA=u}d~Pveu` z=$DF_e%MmQrlM_IH)?Y(R9Eo zwQ@|HZW5mOiX9Y zqmLj38WSn7`&p0{d#nRK2UVLZUcGw7k|*4P(u>4q0YZs`VT7H0 z=c0U8d#YA}IKYgG>gwmlq1T#UT%_R`od%Uq8CTa`fz#U=`_sF>%&Ibx%Ym}#X8Dok z`Pb)l=zha*)-P%n*catb+gM-MS@^6!c?(OdBQ_y9l8hShV^`CI0O!L2ldm{z}Dx`keW0CUfcovo#WU8jy+!Cni68yK63N(D0xT84WZv3u$4wQ_TF zauwy_vH#yYxE)>I@>nv}MS`c`I6?GX(a?z8P+#bCmNZ^yXzfl=87VEVjO|%$FOt!5 ztiM7ny7zLV*IR@Lt&y07h!0v)=yzo#<4bBKXcaU?^@{Ew?_y$ZAfMku`Vv?pzRd24 zkXp7S;ys)&7wWk@)R(qo=CARze7p9m<RKnqg8aKlX7H)xznkcN!qAES-8ICI8~^X74_U8+{&Uk2?*HpYt&22y z?A+(z;FxJ}4p}irf6Q%E6dQh}RP?Gncr$6pxYdt*ukBXcW%XC_Tp_p`J=yvu>ugil zFOxWQ3ob=0?I_qX4`m7tmV1c(_J5LY&eYV^X|GKeYZm_Uz>lLAGvI3TJ#|1N>7EI^dW%WTw0Md@k!+@ipD{zQ2|A<6`PJ&}^l>SkDH;1N)8XmI zR+)AExj5eXDDL0X=*iUgS@0D9VZK`pt5AgD7YA+Cp>#ppux9VWl0ufXfy|C-i$15t z?{|;2n7I5ecfM>g3#w+lc z0*|S<-^E5@7{%=S3gfR~U^Kaq1bQbI7Y${se36c~+5TZcK|xwcF#^v?W}DnKFG(pV zY>@@>Q@+VhVp93v=4usdlu+gypTEZ;)+$fZtFw7jqFstbN=iCd_p2F>Q(_oRjW}Reds;xyhD&J2wQ@D*^e)ZTHsV^%pC$5{n-KuQ+U2CmrKA2^^ z6ONf^>VNzwI3z@qQ;|(Vq~IVmeu-!8oxtAj*xM~XYJ?HkjD^%e-EXDFOlz&waVdq> z?q1{c?8mzrcdM4$5e0BdOH1#!-Ch$|^d_pK$EK&J`^0HbSC^;ItrZ!0E?>nDS}A<* z=+K^A8O%0K%J(a1yWIUs(kFYt*VA8U)Megs(3rw9)L9{K?H&NFVPmV@nf}6(#;sgz z-c*H+>q5CA){5^P=Ymhlt_9i2&d#Q3y}7UzG`=ijdPGVZK3Ve?Z15+WdcAe6Z+=E@ z8Zau;YKq>j%+my%<%o8z_?2knN0yL7BiPzlmTOo9A5<$hc3O44tf2GUoq^VJPwMV` znGbbg2ZLM7zMGqezv5d^9BIiH8SyKJdW`AAA{l#N-y^|9{p z3XADu7ilJ;5M=w!T6UmLKZ$wzRDbA9nO3=hw)^`e9`eb`q9WEuL7e*4GPCb(_D*FH z5jlRZyF*R0Tofhgg_5d`t>v?7_2d0SdQ+-q{0i$kwmwOPjLZu-&KU_V!qZ9!8RUxm zEgxt5AwNI*j4BAt(P-IT#3r%ZSLe7F~!5(d+_pS1j>xR>f9AOSOG_cspCK+ z*0Fg%>^|kZ_Zrn`t>hi%$>gh>>g5M2C%O(QD-TqI%R!VYKBZG#To#7HLlMi0xBWu% zT0-Y89XvwpM7X_;Olx5;e)^Ih{ABGia1b{abLn6ds?`t=yxPaUIbWl3?P;!7 zG!)U9!i%h;fS6U7v_x)>mX=!bQ9bw2S_}0XvK4ym0%z}1N-OVG?gZ!fsrP~T=sYx7 z)?Pn`(yTZqu{xf|@7y0mCibF)Pdc&3r2*8nw1FT5Y_{}zEEh=j$~in-o`8I265@F2 z@M77O(}G#(o{PAZc~fE~r96(qa&rQO-g$JN2=iKUMmES#0k6q#u21JT-l=!dsSG}O0nO{6FLjcW-n)LOBZR@t}4g**3IU4TT6{<(}T})cd7<5pO@NRfc#o9Kq^uJ zr71|g@O@cf;kq`UJ)ICP;-a)7X0vt+auaTU^|B~4EMTmT@9}{*{7@`RVuc#g|-E`kdi= zOd_O=WXfLn-M$Hjdg6!G%`bk%4@O-1&NX|NoV#f7nTqoqcqd~gBY)}7jSuC?kG8GU zT=HqS;`e$+`Y~Hj9K=InlZfMX)qiwf0mlpu)$Q*(olD~TWw}T*}pijWteRoEm zhJqs9E|bRpC?PHI`og`VHj0#|#Gk{c;qwicu88E@%e}Vav9Ym>WG=%e1*`Ps>3XvPrXhUTpl;quAShbHsbY-fFKA#OsJTQ#G}Cal?8$ zpSN${`k!o19T5VUI#vdr{mpsp^Xk>7x#3~W>)A%vH6r`9?a3M+ zkYoIRMm{1|cAWT}Gjw3k7SOWj)l;PcIng%3xr>dxb$xveZ)uzU;<)9y(wonto-0%B zJ#`^*zD(?IWR&r`Y5QwTOpH$)4}*Y!*51xe`9Nrk?}&e)`kQXa(t zMf>bDz$}^+TA0^pCEtQrEv0wYK9b087~2P^=`?_%e!e0!R|SMR%}7ro`>U8m8|-o&U*4R(|9Ghs)?WSwP=^> zY(B|v^w>$8>+4hOhuf3qwcJYFUh)4tU>JTCB=dOdbUtu=?N{os(|$tA3`uv)?=Umr5+XB$dbzJ+^Mr3X3h&_xMU%nWl_yyydS^X zpI#Vfe7rTTy9G<<=?CFxw3a~pjCS&1adGhioC?*z!`1$jxv-Ng;<=~RP3~I-tNtMM z^sM%zg#4Or@%5~xCNaoy>eG0Hl6k7#HZCvg{cwt`;JXi)r*F0}A+4>ga*n^3jS|nnr&kWO zr3^;L#i{06PvPkyDfZqAJpu>Fe`T=aaNSL`?zbiKY2#w74#bICR)E_w1ZEW;QIxKaVZL&!>_{J)vHx-kdfq^qV z3IMgah4V`9BX=$+8K(Fk$l;lsA61%eaU6SB6C`zwdT8b_klTw@=@`O5x-8 zXz9+DXlvRnJU;z#n8;=LY|=Sk=xqxCkee+E-i1dGr7C{%_)>X|qHSnb(iGf}hha%g zh46Tj(V@qkn5bro<|jK27sL!5TN8|1_NOJ;Ws7=jf3)+)(K63^;paDr#wNJ-iPH3( zv-d4~{%Mv*5JC(MOnB_jxw)v_8~MjuAq3)T{oM%fP_6Rp7v37ziOgIFo5=hqy6Z{- ziTG_z4XT?C0tqtXY=K+eu?oF3Vo1%ql>B4Ugc7G>deu!3PD_eY1qc#1ovd9Lm2D8I z?**t53W$;K6))Tn%$msp{P61d0v2Q7WyNdw0_W#p8ZBd8;BCOw*;1U1#Bu{8 zmGmF5T0Ix~5La3=WjVy;$9Qo*sW0EOE^1Y#pr;1XUydk|pyB|ALc@`+0T z3)7wGL4_vZk6qlRXG%fOvAA5f=n|b5qXbBy z6DKCIeSNS)xwKhmHj(uzy|TgbRl=c=VE=e+YY>uzp9zVOP5U6eH{+G+Dw!q;J><5o zIAK?|Fj={RS>#*8nwOiJT-Q|G9U%NX);A1o4@7iem~s{TIK*LyXM~YZxh}yKr`r|#Ok$W#&qZt=N#+>zAU&Xk0t5`ef>|8T7??WR_#5H=o5XwT@ENh9q zvc0CFEp6-kbSv3^vE&_SWBey=r-$aU>tjazsNWy4#V;}J95a1NIJOb;a9K)A!^Eo2 zc5&rddJmzAp_2|P+CYXwFQT9{pGfc4zEu20>mQ+^fQ(ObU9uQJ-ymjLleED==!&!< z20K6vuT4YI=+LXWQAwYTlgg5Lo4=QEG#!83C%EF?ExK&UZEc#BC_X>s&FDX#<=M*o zBF<#Bff2WPc+>Z#=yabq#|k&WH^zpc0lG$J#f6({leMYWn^jRJ#}}{Jc`m4b_bNR1 zMC|DyWf-BShZX~wBSu@>}0U(yCXUZ{3g4JBFb<)Go{duH%W&1K%TpZw zQD`!?H#x(*55;#FHNhv()!_cUtnO39b~e-KH8GENYe;7Kf|N4om7wdXNKQ*D8bQv< zD_oXZkn?M^OeYS!)^GC@>4Tq-#23O{T;l~G$T`*T*D5<}QnBHf@~{jkqQ{KQ?~?tm zZ!Y%|3^cYFsG3%oRNt;lDBvZTi^D0PW>lAOqz(46<(79Hq|ybS%va+|bnRPdhAHx$ z(H+^wlWyO{K?J4}jxCdU)<`vD#!1PtQT(Awg*P;W(Jb51-jvB}n3_J-z`#!xf>&(r znTQ(+W0g_l&P_<`KWg&$=(Am++Y<1xH>wr8mhX3fg6+E${{<`V%1f%kyCKEa$1f|x ztXtWnk*)7*N16_P@)Un$a#b@h{lx@NnE2voT5_x7K!Qk3c(F3pDx7aU69q}8I;T&z zvMzun3*A^7>$oh@BiQn*3O$Q^ey|AjBCpu3ofu8es&8r$Ed*wl9753s z-i*KIQgl{$RQol1pCl7a%5#kdyqvoH&o-qr6XAZDmc`uSoSdd(R{W<pWTU4dBm z{6+VgWW|GnNpErkG@gyt;S0)mzW9P7m<4w}%V2?cTWx_SEL81|T%(6Ve}axBl0?-e zX*mZzQe7Yr{nF+{UEH>FfWa(V!dj)k0dy2&Dt`PdOUW#G2mll)gGVsLk9h;+L7(b%gl_q<+I*3<=9GdDy1FI3=e;W zkNt2Y76xNAgy;oCyRnKFeHkZe%$#F!!2|hAYDz}&@VQw(!Sc)Oslu>MNRxr}g%BXI z@~pcpXoyUTn2sI3Gee*3;2`^=KZph1_#JDe5kfch%y>4|5}owvR6|yR*_FM2Np+Sd zq*Ghx(h0Red-j#jLTRu05F#1Ruryz~z@_F67bOA}RSOQ~h7OG7q8 z2Fw~46(MPVx-a2RNHmWme5gg_LlIvDpHU|W6e=_YJ}J&unix@bV6AJVn`)4$Jn;Ul zagU{iFdiEXsX9njejZzOsMm{}P%iSDv$!>_XNw$bW6wW;a5duNu<@4@y!X)LtZR1J zZ7W)x7+Fiv;b82?&tEn8S~arTv!-~lW4)xw#3v@UB#|olL z2*x0|)|`ekeH~io9gEb{rfg%)Kvk<}6cHC8*ZP!^ zEI|ru1t+8SZVClvV->s&9^2*e8n(}F68>T%G3v9OYL=V=40FSc3vdacEJ7iC;vpvp zdF}(`jQm{qc8b7!LOLEfjWqXPKqCe2o|w<)Z$uf9)bR>LCj?Mcb8qA^^`9rg+kmB61T*D~jh~#lS^bSHS{t*J=S0^T% zk;#Rhf$GfS^5C;%+BB0g?x%!9DQd><{Ubh`G z#j6Cgs2#i$Ve-=+|n~)jx<;6Rj<${O6$Fl~N0sownvp*}4ax_p@I4Aet2AaaQiVRkb54lzr;HGyGTRV^K0W`= z?zeIk#eAz7#p#+bLgrr|nKRlWInPHOp#s+_w~rzKDlimZK87l2}73r z+-*Jn33i@Qql*nWRF&PV6H>DL-uj&;pWm5N9RUUwuGeBUil^>V$rL@V>yqL-$*Lnk zRL>!@QhK$=MhjJGlIFBJ%@UT_%yPQPbhpOJ73-T38p$N3umNqP!+4Rpbbf-0H3eJ# zdGd-mF0Ot!!XW7o8f&Pqx8!Yc-0TAFNnns4d1f)gDne?ZL@7cVoevAh6qskre?PiC z{Y5ojK6yXeT*5pMVYHkf>T^^yRGsM71WM#FK)V}e&ht5p#7W!~X*g3HR%c_|E9IlA zk)7^y+9i&jGcH)Nc)nSqMc-q0>h1}BgQ-C8Ti6qmn|d%)n>F0i18|nORw$hLqCZ_| zOH*1=Kg%h)60!2;*P%EmA9qwV+sfu<$((JmM*h)KXUwp?oLuY3v#irbh5mD<<%yo^ z>Qx@g-3ri8Zt$U4*1PsrBp|`1vaf(@nkfX`*q&MJ`BQI&YROa#!bt#E+&_Nou z=iY4x-|+shOct^GwqENi<$P6Sr#*kgJyyGVliwBRvbVye_6>$*z4$lvustGw50vH zSQceDRLOdAo`Gc0tb*N=Kgjs-fpu%h2AfgYO@~47o(2Q6V4OyXvbD60E~3C<{dZ1D zBMqTg_Q<-Tgjq3D%44R)lL>5~ZQWDr4`-}F94#sK>nWLMR0VJ#Wi;+E&?!4ctp8m7 zm6~^9+UG=_DE{x}b#f>>t$!#2OPS*8qlp+@@DUsAMh{7#d=Ubff3mU1P<+C4EKJz) z@ksY(CB;-LTR?#Z6kjqjSo)0-RH?~sqA4u%FlGbM97W_E=yDF8^0(Mk8S~=*Pi_l-}Df^#Y=cDVAnCanN&Qfno|Bko$1j5al5tV|`Cfn0ddh{aV>5)Ft;ooEpj*1TRO z)s$R#gV5x>{6&!-ms)|U$8M_Dkrf{n|Ut@wbI=hVqQ-7YsTEar6^LoI+-_J3>TzXy6JH3 zUgtA{g>}Wzm~Br~v18}~_M^)8beAOM8Wh|6fXB#DOyNEQJRBi4bu64kwM69rCx(po z4l#?04A4Z?>TI+D14aXmQ^@^R@vFbm9WFo^k=mK613b_>9Af68SN1?r74|*hjHMLG zt^SA_KcR)?zBBK~>I$ypP*d6Fq(_ux+Y4_WMc59FSJ^2(7Z@V_zOGWb_$q~WFNWVJ zh*7_7cfKtShy#BB-N*ik*K(ydNw3z5aIC^O3h-GHKaXcyd`UG6l%%r7{hzw7DhZZ@ zF%)SQ;l~935|KdTQHs~2PZ2hwB{{DOf+EWLAnTu^&H3~FDA#Ac z3;g@?h8?TU%2%XwWJAaP4xdON3^ zP6#D|eXQ3+@punQzw`OUJ(Ae!?+%4uKekxAhrN&^=a%##2MVsb@*Kj$`t0vw2{hw~ z9Bd!8YDu}YnZ|~OPv$+ahyq2Y{Qhy9G=#ND61z6cUNw0%Id24R8GrY|W38@jpj7v) zDoXI-m>;h6Ve|0tJmy8cM{1ccui`bX?+7KY56iKmJ!TbW)GevAeq=m4m{$}`8GDD!}KbicB6|XP|H7ZW_DIP_by%kv7f&`+~UzN z*578&y{y-b?}y>s92kMU4bL<>k`gg>pu{3p?}A25diuzU&tcaJeQv)sRHo{GEds$q z*l7Vxr_!X{W+$23I1*%HW&^o2h1DLoJl;2U9D!erZ3XO#`Z^|TRF`F(+}tX7O&k&t zD^@sOD*qFRM6M~wgPMH|jiM^>g-V@9{u+hwi`@M5Mr+Nn*ni3`!7Vo;zk4Qj&)qAS zLHAm{fAD$#X0Rw|z)Y#E21mKF`ruY3ZtK6@>vS_MtcOP%C*5Z&F7JgFG-n&DNJ ztMgoonEn)IbFoy=rYn6-Jq% z$msVjYXkNZx?G&=p#7aZX6*(busFfJH_|D%5iCto3oGCmuP%%n6{j@dovF z#WwYf*Io+mvYr)L8St$s7mXZ1C3xAs0^Du}6c=GY{YUl)AzmQ!l#orFD`Jqp>By?V z-sKv$6$a}ga@s0NF7*FR0s<_hza`)6-CfA>MB5bin&N4Gqo)32y~T+U!JF%@)4k?I zUFa^P#{7#s%VaX1%W!I(wnwb4*t{1FYD4*6AXAZDK9;iNz6$IaL`vbX0i})-Zw3z7WZKjr=_*u#<_FB+F-{zf(LYH_c98E8nA(^0WNCV_Pk$<_b>&RpN_E~|$ z8OdcP4V}vD;fqeL@lvNqk8cv_dwHJb`EO~SSO;PeFXnxyW ziw-kyg8#S=g=W)R6RoYT76hb7nFrpSzZFEY19%O$k4oHc;;AWwhKz=WCduFh8hq$PYk-Hs)XAgxJD1wCzIrKJ;S!pZ@GH>utLOow*B)JIx##-=lwVoYx^A{Siy`+-U>JR2O?h-#)as^{%@c(H1LE315H^ z30fgmA;Ao6<3Kb^b*Djv-|Fge?0}9xnlhaS9J+5hb-A%3(KwjxBjP@OKZ zU85}f)9US+D9I1Hc@ANVM^SGQ#{SmRgqs{k(KGbMqr))y+S}X1a9t^c9JI_HN1zmh z0#_h+YO=L36`iAGa{(M5gm)(HSxEx3F(^{I5K#EYi;c_bfNLKVXdiWAEc+RsxCrEM z4WZ-fi(^gy_se^|$y`x@^3R{XOnE|~$*Ya0VY4&Ss41wNNDDQALMQqE*`E{wbsesw ziVZIpXfDy#jmusUyFWVre)#EA3V*=12Z@KriPhtCoB*Rk!C2XyPX9~2iOJ3mo|(rd zueH_kuSnOm+qA;U#CkuYBbR&N&WEeY9ecLp71}&>7?BE38I`}F#A)0o0{4K?Mzej8 z!m&9g%`piWP=+p;RWgeBilMV3rgW`lp1Uh6IQt-RG6o9w~s5WOzy6oF~Y!OB&L_z=3;d9{gj z7znTm%_A;+Ps<()i^XALYb-HVlCl{$3EW(4Ri;ddle29621LJQ+cWzqvK?vWh$qkN zCO)Mppn#M5)P?<)d4{MnMMc%5`b;a)uJDfJxfS-2w;AOe>#3U;r5Pg`%sG}Z>5F-3 z-+H@x7+yT98T^}kXJdkMadfeIUYNoMx;iBy+MU_2x-IW<8#u`CS#WaDrS2`uq5xO1 z=b!K5zb5$abx|^?A@dm(uzfpt=N@0rOy~NRCg@i~$Z6G^!1MTUYkN7tdlvid4m z8c8<_Q0%Ge)pW;^LG?U2-v2CozI@ysfK%61{WmXisj^Sx+je}=^UG>pLsD+r`xtza zxiqmL=E+YGq$ttYCjtm3rKDh#ge}y@elH4mZmD^5YC^{@3xx7r~!G@cNr27CmuJnx^m@}1AAL!=sQ+UH z%*ExVJtGV-5+?d;6iYWjXQ1r*duuUahY+haD|UNMq`xJK9B_;%l%0#4QGdB6(YbJO z4iWBLdpXqLBE0EJGMZ~VRJlu%$e}Yl^Y!E6e|?C(K{q0eI5lz6KlHowMx2V?H4|3+ zjX3!>F5;VOywtin*Mb}MAETl=mdo`3)338b#ch{{cf#@@racQKsi&(--M z8NX$;{%Izvq8NF6Ekga3hf2PkWhH@1vi);#x8>=>?C}d~ZG~}*cZIxQ)9Z{^nsx!p z-Di&-1C&#aUGIsYP2%f-4P?Ia`GExqtJ=4x)WsPP3?*e6t zW8aqn0Ra)g40Qeb0I@`@uUi>?_29}p{fK(yT8H_z*u~+dyum6dc44B}523c)fzj&{ zk?rcDW_U!YNtiOgUlZKYA~yBKF;2vFbzyxdugIWo)Ae>22=AM3PqsDoQ4^fFzy4=w zf9ardA#j!&zsZl)iuirsE~z?4Cxz*=)EN5nKCsVg7{_Q6CMhKXC`q}E)UG2?&yE*z zoHMV3K<{E<{Q(w&PD96*P(Whk56_|EUg|*dwc_uS6a~G9r$V`QZ$<>#oJ^HC`N(Q~ zcP*UE<;tXbKh)#irOxo*8P$^_plC4>)Q8WT=nqtXdrqu*{g&w7`Nc&pV3Jf%yL#_~ z8mt4EugGiBOENJr!FWlLsS*y7GQF@cSjAsn$Ns1SmMU!M#clY~N}sTUXIx(HEfW~FP&;<5 zb?EK{ggMr~;52y9j>g6SdqP3{yDxCiCVsi?XeK-Jx#u5X6k&(JQmUr%5$G_)Q8F(a zq`MEmOwx97gPOAZJc~f(c4YbkHQ_Sq>a891BdGes^O?@VI=O`y7iBL_PF(Q%>yOfE z$n(ZGw3K57ENZG%=69`#&{nz=Fu=A^c&4fmP-*xDADERw;c$2wb6*U#*b@{yh^LcA zx|^cliOJ81L*J+V^SZ*jWH`clbR(A}HW#|m?}DkxhMj31jK1WR%ylCrCoc~E1T4KJ zBXiJ|flP|N6rL_%uh7ayfTKxe>bD=d=_}yAsYT5TW~JJS;L6c#47uNs7xis_JHaNc zMkV|U)xknvfN961%B3S@}ElHTa>n?`Zvoo$n8Oyp{ZKDAZtMW_|)EC6GMxS8dpgqz~_q zK6^eZ-_kHcb~fj?+)szM&C2jU`RIP>O;*Iu@wLU)7>u;_?1Bw$s$|mg($k-YNztcO z1<^n!MWxswMPWZ5p;ekEVGOaagIbg#It2%by*g^{x9VA}07l3GVfydMNmx?SBVb5Bn`w)PiID|Bf210a zDJsuQ3x^kh4NTW;7jm)TveH}pP9f$Tkh3H8#>U2@C(k{1%QV!8!9dhI9}=FBQQX|1 zPZMx{yU470Ncbq(F4dpq*{3BQZC(a1v0c$0OJ(EwG80&NGEmd2gN*aCIAWj;MNjY#~a+YbRrMsqsiU@<7%#a95o1a>^Emi1a9lUrS58> z9I&q&!7F@1rJ28fzoA^ppPA9efoIkk==~GZrJY46aA2PEZs?Rl)GTFG`TeA5``{SC zxOAo2s-r|svO7-n`e)2jn%Q=qHr007%kYL}!qS@oF_UnEERmtTrQ7z&p3xb{5Tst% zgLp9h(96?324M6O23D5)R#|HRE3toi!e@R@E|xOY?q>F@Cn}sBZ46`cza7vTZt~o# zz+%1LEUo-qZ*K^qT+s#*NoaWZ5^#C3OA?QkXlr6*DRAiwW{C|ii=pOmXbW53Kfsrw zi_$%4Zq+(A^z9w{vd>b1IB{#wf;7#UykytpdhXH8cr?a55hcL2dq#TKg>;Foo1V{z z_&GK<_9M!cGJvRafkkY6q!>PQu3^Zi{MYC7=9Vxvf3--r(xjww9hk1eHf+br+Q06# z+TM$m3BkfsE7RjGGip3Ynn%3_VSYK3LsT3u_Q&wP5#IgK#&#Z^IE<{Ev+sPmpNx;}@wW|5e=yQoiUiT%J5_j!1;zC=WUj8Y^NUAx2eP2jVQ$bC1t=4=0k zN73KLsS0`RJs&4oV=%Uu#DTIY7gW<;OEE%DOPzRmeNb01Q`32K^hALb;?vz(e$Esp zAEX5a1_mJ&Rk=+A4EBxPnm!!mNzc~sW#Tbo^nSmca;4Pgf~=R_&`qu`Yt|4w(bqRJ zq6BQ}sOlRLgFApgmPZlD$Mw6Jl^2xO{hM}w-r2mnyYNW;j*gFQLlZWACo_pfM~3r| zew@)=g_nin(p`HSxUMQ`2U+aHSP&!&1jbTk(sI$*Co3oS1kZ*sJ@c8w;vgxPt=$bk zO1zf8$nvHMaB!p~C4+{%fgzB6KP1D+TFT*foqM5Hhn%%$a&4m0Wa@84?L za`lu>aVu8n|K*5!Mxs6-i{ImZ=VvAPLbeTAJ_cUGH%y4Mv z;d9ZDO!z-ig-_DwU>OHQ(*hXha$m|`W1pyqNO4*=A~!b|_mKF;FVzALyHSR1(^=zb z^d)m?%Qb`c1oG?WJunubX#?1Y+BduDzfrIQc>i^k9}%CN^Q{8B$7)z#CW;R_Af8Pv z4`rCtbFMTf{G(PjN%h%ipm~nFuKWyUBGsJ#IQD!I5s^u{iyVoJ@(||<<)ZXVMpR&q5vpRWtz`$%;L}zAZ`jDI1+h?v97F#QyAzdhV z%qFWX0J8kF$qzz@APJiW<4+6V$1D=}BBJdPPj_uIE#$P&_s$2@mZ&!?ZCC0_hf6I9 ziqT3ATU+k?>VGR5YuXSS`P%*S^AubHV$5+bs>skr24UY96M;+*Q@9Sry&+qVhv6JOP)t>8hiTp8*Bm2g@U;cTn_rBwVZg9=@@Gfzp zphYj*Mifu$H{gIIYC*XsSKW7}w6D3PU{*u9p^?J*+3|8VaL~+B8k`G&_LBmjTs%pC zX8(O>da&d*I$-57K>vlp*!x(MSWFqv^IRC}oA3KLNGTkeA-rXE?p zZ5r&+rqj-o@YQLY?M})p55o*Mn&{U);g!cxEGH3Pl_32ItdEvrhjy~&_%+6o=+Y;8 zf-3%(HW3iWf&X#`oUlga``Q8zjZ#McKmkCE$==>v^I9Vr-HEv|1}jY4sDb-ZQ)mn4 z@)nmyoD$F_FI#+$M`D_e2w`n-ab93_S1k0*ZiJA30mP=Dr`>#O%VwP0(fW`%2HGv~ zNSEmR1-{r$(5us5--gN{jW;b`xQe$Q*((ZQsRZ?P+CSaHE2vmJmM})PdZo=N3?G0D{!fOQp7R@< za<YqDhSMW;t7H=c1aV*4V`5W|cX*n@QJP{Zz)zxiJINAZVBq89C-ZLpC z7tM1qnNfi>k~%-vVfcBwJ8ohxfb?^q4-fIK8;(eIihL#h?fjPv);cm;3rQkMD&LPU z(oN*5f#orY+Blfeo4nK*hGzYDp`9HghjH|`4L*%TKv_WVykh5ffJX{1Z$LUs@W#ZH z_*N@x+94_|^unI~Tjw!4<-@3gXBNCgOR7%J9!ohnrG~Ve`L+M0cgFtAIvD&gg8-Az*+Am-u%T0Gx$ifEWl>mO$?#VHo#~+Jk3jtWpmM&CFpzwfc6ZNGP9UO)IP}r;_N4owXIwKodasp%FLg^g8Z+j zlBjeKAqNFsUY%RNk&VwT{6(nh>pflwn7jM~?hF45+9qy3AT$R94jayvC^FYg!MzMZZzIK~FA&dK(ICPnooejSd)5rFMS`aXi*wGtE| zWg-4QpgK$pjg;2&%)g;)dZB~8|2IPAx$7Ra1^R!GdiOSd=1fH0w9YHl`r!}p3-^K? zGw_olSHH3J%cCfZ@m`S!Hl>DykY`6+Yr+zMi8PJB{p+&Bk5{34%`&eBzBX zQ~T=u*R5~hWp4B%+DDvvf0K|hiYEf7nxVJJ`fK2vz!Q$^@jp44f+eC~b0xjdy*A7C zhba!#b7YiV_vkIuE2(I&0i76ehY?D34a$|0DqCQJ{s5}a(hX2_e*+4%mgMnIQq$&L z%YQEtGF83iS$5|6Sfv3A-9t6}366(YlxS%0VxCUkuNL3LpDi9iiM2IX06!8wdK3yk z4P&49GoK@?krM3(AQpJd_M@r=bu8DlA-rD(eAFke8y9!gsGTP9JTvk4L|WgW@4RiF zn`;Nv8zwvsUlg$8N1%x&T1zk3i0iLW%FJQ6C~AI+q`1FLo&RJN1d)8T%|( zziwysa5wzXURhT>z68H8ebnm%H(EXiWe|2E4=u7drNIv}sq;-#ho9FsIgQ+ZV>}xZ z-t(pwF*8#8;$ko0_;hOKt9;@Kx<dkDe79V#_$A9tw zXLm&4>C$O`6d}c0ux(?u%3;KCPPs0F-;~9jGJCsUEm1NX#I*exBE+BrMYhgnRO_=A zXy6mPK3M4!c3Xeg+|mNN1abf8X_8i6Pnj!kNK%6TsL@wvr0Hx!=}6p%&79J+7grX| zc8wVo=Fy5l7Z*wjM%D0KhV=`ezsAPr-JQF?D@`R(+z-G$0>}SWs{7aDn{x@yw%g~s zlltvb9Z$SF9<=u|<6h67;ntNr<2~IaY)KR#hfRQ{P!uK3^KaA4+|0G%8=g3R$Io8+ z@Wog3N-?X!E3tGTYM`Gv)%WznFPeXC%<8MB3;Xs;#v*7cH~EXPSFF@TVTQZ6xv9n$ zev{t#8C1ro(I*qq0zo!V@AE5kRCnj6PqgdD7cz$yuO@*73GuUljTm-g>wj_7zSt4I z51NL{0EYku^#jx_+BN;dg{r079{^I93BCX5SIbu$yU(C{0d^dU)wxH26A>T;SR&OH z`;GBBKBlPQlKo}#-{^grc3~mEX(y)pD;)i>LfpP(bR>*P@prkQ5a42RK+*B{;6R~X z2^kcz?u}}1HEIhm#(TNFu@S+pT|)cp*<;Wo918kGYXIZM%nHI6#CU)0_y;u=(YR%@bHeGq4xRUY!XF&xZjni zo&9HU0$zlu#CWTh^GZ)@*~LHjUHRJ%Aw%lNWESkITe8YUfG=Z*(EE@h9U)kcJUl(a zOG`QC0xs3Uk74aiMTjum{K3kPt8Z{2 za{#Cl*?WTC?ae(DC;*)v3;@t}rwdiSZXF3~JsvY;f92xpDh)Uyr`?%G^UF3M6D6`~ z$_ehnZ&Z;Ik>C*5=ryhQ*6M3eakv)}I+(?1&J{jf{fp*Kq0@dUcKganCKc9W8(*N3 zS>j-@Qt$=%w+c+eEa-`xAP?!=6QbjouE$WAFqjI*$e_+9-{u2Q)`EbZnrA&+!2SUy zaNvEg+z#sQzaWNv0=5O4wE6*|c|Xm58d0~R`%XF)M#V7EGW{AXz!?PO`o}MHcXeTd zVQ2|L3?xK1PDdJ?&HO-%5Ig!0V4qMc)!};d8ye3HII1v|SXb3%Zi=Yif~$%{Nvu#A zK*pEAvAw{U;-M9B)U-%e!Oj3xseJfM|6lbuLZCCC;xy>W3=a=)+E1eB0vWucvy*v< z?_l``xY?c_7qEq#gLtf69SS5(Y_Rb)hEbg~Fki?wMkyGkDiJMcw!+C2b?3XXcXO+~ zudw_AK1Cm6UI%wo;hPZw%w#YvjzbtOIgC>hX29H2RFs+b`t@t}by$Ehu<8hLvY^Aa zQ|vU;xwqioluk@-Qc(sSr*Q<7m^qpj$OLS(XXve_(-`RJlLAv~^De|Z{k;M%@IcVzqb&`8y6`(K56Ajt^3#}lPM&G+q>(8iB?w;h$hK8` zLU$4esi@ofCm0VzVQ=IfF76LNSn_&o>%9*PD@l>F2>}2g^sDDy$rO_^I0-~||AIa{ zc_*jOB~^f|se0YSsR`or_EeqXbpU8y)4tJAWFo+H6U|JU`}|t^_?2X73@Sz5LxG#a zF10e8DcnFqTVZbrnTqb_56*t?9c=@n?&r@xJ>@2{m_ds$YeNt;Mx<~$bj~!oVgoNi z4-uJ+tStGB+)ZXYCI+9X!wDRDBWG(+09-65OWF`u0qZ8_IzDI;Xd}E!2?SjgZ6qen z1Y_tG*nD{=oe-Ds_vo><;chNPkQ1rKxFLvHsM)RN)M7(X?p;0rAI&VdMm=wGU7mc~SntpXWt zRUpYp-Wnl^i6=r(3~sZ&G`LO_gFZ=qXE#}~wqkcrPtUq2f<^1`9-9WrApESB-bGw(W$OvP<3?cLJTi4GR?Cf_?LESRJ($g0ixI8FvEAYZPCEczE zv%rETZya9RX!sJE-fg`cCh^+!1sB;vBni)J*XH-IXN!t#6A(OTG;QiUw@LFyyDamZ zn)-SWhA0Ym4sPW>QXMR?RBAaO+mQ;Tm3E zH5Ogf9AozP0m=yi=FVuBeExS)n$xge>8`DTYkdlnR170DUdan7&zjGhlqGMIW&RZw z-9F)6QZ9X=01ava-R7ei>I`d5RRNv=JtV(coGP}eDH2WoNhMtrBlyPt&ElT(9l@=n z_8;hNe8Th!QBM#cu@_anLZl9VdxwFyk&?gH1>)XQPymib+yvg5rH_?4fd8_={we+y zgs&fZD2U_0C_M{u<*mY2hM;Y-fhnl}weMgRYr{Tmfh3XN%36jnPim z{l>&jYS;V#9e+HYUnkB(#9(Pjh2eE34kAg+_N0=^608b=(%C1vZ)G6vi4wK~pF1C-t z&bjy3nO?zv;KMYG?$E`U-!ZDQNm9@i2QK4B(bvFHFwx-5;ITK?j{)cbOH#@UGPus^ zH;-BEkr>bR!S=ilyN8R&w#9!>j_`DY7(Et1D}K}v-2leI-vso_~0E#h;=QiVJa3J~g_h1L(eonpb^xaqsb;wYI>da+NCP=`p&9Ca z(z=~BjN|hm!byCZ0HsiZ*Fjf<>?;^W$F(|=Bf@!w8TcOIMenxNcaJgV!3uWvPc@3+(2_F)tzv|z9KycSCZpQ7JA+He*Eh`4A*y=OD0OqFkOC4 zHS^lfH6JBdRJ;| zsk|`Z+@Qos2VmjEnOq`Cx^+8Rl3dfDvAG>81ClWRA(+I*az6SMH_DwKmmLuky2^O{ zH$X1QwiKS8o}eP8h0zbca*XZcfrPs_(G|b#7#JCOOpIu^+wwjvrux&p&lH~IPbRrbZQmF;3nf7cMhfz!$Q_V3@n zAvp%M&+eXuhJ|Sr$I3_JxHJD0iX#)+AV$+Xj?Lx*`F`$ksl?WprIx1Pv%a4HMb(}D z5bVfRlA}DtPyC@y`J+tgi-c#AahUuG>lFuymk)`mc{@8O@xHvo398PN4Qqcqy&CG5 zzWG%vzLcdQm(ShT85ptx+m?#vB`OqG`6U!~s=?wwZ`e=)Lo{qxI@-9Cyz2?;PUUh1 zi+6jxQcWmtnE^%35V^uJUGHgV6bUyA%RRi0oB~EU_|Z%E465h{I8b^x0J63aQ4nOH zH3j9>`k1A~Gsiee9d=H=Nq!sj)b&zoop=QIEnv6RyRu)ZLB6g|#ltH?R5ZyfvKe{f zUc^H+uiqcFu6&smpSMiG8{f>_X5wEy!CnwaN|Ks|a)w1Op=e*X*D;~+-qZB!@v;-4uay6H>6$4cPzaj~$V1l#orv6Piftz8 zfGjcse0*<{+9nQ>C<^LqBOPls?bNh11TL0|ms8Zgrx_mIoqlPcz)U0=zbAV62@e6y z?92Oe^--b<_N2o&k7TOE#Fm&UANbX~F-+sG(w=|vx1eEZH!o^I+*u9uN?$nWyB z+U|jtmKO6Ld`DuDjsZwP4ucoUv`;XTyUOw4%DM?atZSgg?$#6n3?Z|3ap|#E@@Hn- z$E?yZ;xD=Yotwe+&s9eG1g$#d#(rqua&5!JhrTUQ*XU< z^&QR$UNIPK>+DKi4a1W0wZS5#mZ%7C`}Ql`Av8jE_~v!50->eKTcn5Mec~2YmR~lL z#cxL^CtpIv$m5@d0K?C36{AaN^_b<-0MP6rN5J2ykZ)AM6upbjk}J7L{k`8YJ9`T= zdhI>$1S&(jrm1bvhdfjfs=AJKI5jmju=Jv{Ds1lsAG*~F^UHVQRtsTlJ<(86+Apu9 zJNz8dN&Qn7Ot+fbtkKU#H?@!d^fAuoqR1=e5moozg$W4O*PBMwqxuyV{ciL@dAhpg z*RbDJk4mVqk=XDe0=JCAzxgkML(hidZgAW zDi{m|sd0TDti2TPJQ1Ne;-Ozr-SJ4IY%$HNmY2^{@avyH0dX{mf7W=Pt#@ZAc9e2C zGbj~GNACyWka4N2OpX-l_yYkjf1O*m;x=?38ZZ|a)jj{3bW6)FHs0hZAPF^2tKrAh z{6AtS6986VWMjJmR!nh}J#6p4yuyL;-HiHq{AUG>fH=nprp_qD%Yzo7OM@X-!5q7Z zR0fhE(N&-|*A(D^K&+*bje$>nsJn$EgR-39JO^dNVUm6C= z7hHwt?fWiU*J4c{7?0Y^I52Okp-psro~9>G4?uf&Cy_=7+bclxddHXWN;z2;lk9KC zsAcj+-+^Tu)>rR+R(HIxn^+MW^Z z9Z;6vuY{=l0qu&WQ!FT&;s~Yr#uVbQ);sq&px*uc#X^&6XK$@vrqNJ#rUjh3G%KGX zk3JwgZ8FRvQ= z%}{cia+<2I(#|cU3Sy-PcYal%WRgi^GK=&UjeaaFQx@@@sap#R zet>r+dXyt5CE4QfG)`h!?KPg(OZIJ&y>oi>d1#6&P94{-`}h`dpWqW&B%!ba;KCuV z*ab&&xaLCWl%!(*K-gP*5R)o1f?;TevIO&&@T^}%CHK7wst`Z-H_4vYLf9Hnnbytt zAuKw>C~Z%KShvnrA%dH|3YvA>jSGM_X}Gw6b@5BK*kQ-&z~J1K+v88foZai0XFd$2 z$_23%uvh^H3Kt<9V3Ts>HOUiGBStEDS_Nb?h_e(XXeU;4OAzj1H=C-#DW=5Z;g%vN z6Niw!J4c`YbdqMuNfkfTKJL=MkdxJs0IVVt4EyqR87OeN`um9x9hQT`O{B|zakAe$ zt|Q8hpvD&Z&F>?4O`A{FyytFFKYI~s^Kq2Wk`$9xj`MWpsZiY~A&ik?VF1>W)RJ+p z{#aTHDE{G-m8uE+D<$yR*NM@wgI8X~Da=V) zZs^O#Md95o3ozdqh=jKH_5@DSOiyqWhS*o1&Qe`}iv}Dhy>yEG|HlQyvOMI*{$1ofe3|LtDr! z4$x5LMjhmU#~g1)PzXEy;#k<@xzfv6tt=Y*x~k2R^wYh@4@Ei?vPrRB2%jN+&6VOL z56BaNvk+MG{4V&UO3uGFcuz|fam2&-@4gztw$`mRmKx$vCzhC@>iFs`o0Qsxngw=n zCNRl71n-^)J9Qu^%Kse!hi(e!mbw3K%dYO77Jpu==LELHX>wWq0);0i&} z%^tK}>U^8RtQ?D;D1Mug{|O-gLMWi6fkLBT#U+eNNbu`yty3Sy+v*2glEqtD#d9B|t*i~2Kv67qXz`q#Qf1aqNX5PiFcc&5eu^L_T3y@QO0DvGcY zwCOC(@ts|t_%@}n+_DoQV*ba%;>%`z-<()gccW?!?aTGIk8{LUpJR{Py==N3#8_>A z31B`Vi+vOYj1W9LyjQ^I1bau7T5C8p83}?uM`-%>WWL5?*8)y}g2Qo7Y+*;JJyUla zgG+%yv44;=Fr>A$HBb3DK=?w-)<`xq?Cy6G`)*;VMP1{HrRw;rlQan@`_7B)I5khM zU+%HZK2JQ zPpA#41wA;2s&0h8oE~Wz8R>)hmMldgyd^*tO7e(<@01M=#%+m3mnj`KA$Xhj@N^)R zhwS%E!tZfU5{;Yo8u7$LI)=Uva#gS2FNQKMxU7bmDrauC+;yIM71gLX!)?Kl!Xk6? zd;Ny?(1*j1$f4ki;-_L06cwd}j_xyzD7Jxk&j7L2s*)WE%vnfa9z4Kp|6af(CcA?O9695SRI3`Ijo>Fga;@qM)xlj9b?j@UkmcuE-01LORnxu zUK76GR{!q#*$fvKLwBNp{G*Th&0<>(F$-Sc_Lli_um>9lrzKA#^8sF5p^|3c4gH$$ zYh_6Pt=l&|ti?T9>tqScPF6kDf4svWu){>0Pe1C5a2Yk7R*oQX;I^BsaD4g+-iyD7 zkxKQ-Dcn&wZu6>XVdh-oB38W1WZN9vM;E+k zh>jaJvC)95BSgO925+4MP;Ev@`FJBHwt#vI)=dGkSH0fF&dtru_)7p5M@h1{J73>h zi8s(Te4a5}#?-p^|3gYN0@!e#vnU4cPr;bOZ*vP_XV>skr$jkcBPC# z;=l*~HQW6hT*Svz3#T0LUFVG2mE#6SL9UnV%m@4i*uw8#T+17>*6cgpDR6(iAf8Xvh zJM{{sK;G++3HNAWVW{GEqr~HO&-b2&nAFb(^Q}q+-*3B0_(@KC%)jATOsYx==G(em zs;RUKe7S-IrkjyWSHShQq)Ii$Bq2JA#ZA|#AwrjH%O=)Y#Bpn; z@j>w|cC5l^mS>UXWOuBKmn`rd9UVVZsd*Fk3}|jQ{FD0SS8shyFA0*E2_K@T#6N6n z;(xl%(Zc=(?^nM`u(L-nhsj7Mg^j=6IPLsOT;_Ifln`FH;YzC$paND4Tyt*+Vh;wb zi0Q1d_T>$Uy_oQB4`>3fEa|Tzf2g>Qzq7t&)m9b-x<~lv!P3k;=~pzO6xRc}G?@ zAbI8I&roO|&48Wt!ADc~8MqfqW#41X&&$h`?dklMVDQUgH0l44f$}Xt{=r)jO=5P4 zb6&=c)Q*q$TV?L)2X_Ev39;*Qlnx{$<#!oWCFsJwSVY`nqgeK{&%poE!a+R1XjL6_ z<4#7={9;tE?d&TGq6jeW5(kjXKSs+p5rPxkEu!oNdO2!T|^ zPy{MIxjpx2{yBv%x4Wbq4{o0FoX0n5^8J%}4uYA1ZDRVwbEp+81gH-FzP`I^nUUvX zCcHtd!IMXwTDaBB*^ZwcWF$vFcAK*QuAYubS_)^=&c2_kmpDh=CsZ-dMFR4W;IGX~ zIwaZQU}?>#GGN)mXN7o%jsZmwyPF^88=te45{Ly{V{~~!{=aO)9FL2z)Rvu+ zyFGqdee?8~U*Gw4n@(|ga8>wimsz)~A(b5_-`Kn&S+9vM?YBi3zJ2nRmGy7JG}Tkl z>lkQ9873WDy=P>>!2?wo`QS-K7x@V(gwp2>r>=QOHdCLCu9EwnNpy% zXtj+;mFN#J)O_+EJ=QxJDM4rNt1r?Gi<5}osfzo4ExiImn8G`zl+Nm ze~nMjS;pdT`emvQ6^K|LoW%no5qpHnq|is#`xEGv@rC0_p``r4V>tylsJJo=K;lKf z-2-PwKN=Jm-oeO17(l4xPmvp{Bc)Wq7zf3WK%#(<|>)l=*q6 zXTn+E(>u8C+z#2mqaX(&7gnS>oi@dzJftFU*NtMhJG2XWl0_J&armd5#wOg0W<(4T zZ!|(mK$Mp7lpiMT8p^g?f|4@PBgqR8BYxjW?)u0BZp+~ewg=-p3Gjp$P> z@sZM0V+B8Q*FysM(XT+HhS1Bv9BI82X>Apwt33bG8ZA!v^gD#~yc%%ntJ&HW`p{96 zVRUB2c0WtB=S^^#I3P)k_d;Pni}~Po7P=e|zP=zHyPu%&iU5MrJMi__Y1)RHG0^id z)8^*Zl5nb1u#DsA0vnY+nTyBudrPNRH3cnBZ&%OfaA>>KIC4hTDi{YMQ}&G+IpfYYSVfV8M3>Z;TEfbd;v* zJ4c-f<%IC#*d#79>FMbWkIKu(h~`TmrlclK#l<&hq`d#(JN>tQxb(L9ahCfEOFbCT z3dG&imB^x#^eVr#p=Hh@Ml%gU}?i3I6M2!sSGzjr`>Ac2wE zt*_@$aA<(_$p!h1tq0J$0NQI|_muNERhPUcUbjf+$}wJTdz7x?n-CNM6jN{_!jQ+K z5(K^*Ysrff7dHD{S@BSZ6V=Ba@3m3 z<6mgMaQ6{C{}b27YIm*NDyEq*?nPR3B2bA3S*m& z*zbv{FF6dZM+!ZX?mx%z5UC?#8S%*BXI-8Nz#d*&$jz(FB@OlLwU+|PC=|@(+fo9oPARkA< z%7^<%8^)$IR&(%(t7aROM{G@_H=puP&}xutAxRS_AMV!B#*<&uj11imK-CRnH9WJ9 zaINzyXj*}qKFpa-nFAIaNu~(&m^86|j&Jh!W7OD>XS5D(qMIKltzvh-h8NeXnoR&; ztArN`Tne;Dj~ELf2Qwxy}Gy26E62{R}4R_{ODfW1>IfTJ8A1Er2Q*x zv8*y1c|Q+<8+}Y?{{5=^;XTg`?dB=+;lDS?t`TX7^16$PD13qKAilDLz*^5tp6mKt ziWqx=HPQt}mVvuI(~?5YyLX@K+=)?jLN4E`PxQL(L|p5b_w2&pehVn9p(%NdFw~$A zg-1c@E#)xhiyaA>F%Xb8Q}|fxk};4%j0~lKTD1En!Z@mQAq<~t3}JY`Y(nr({R#cj zvn#C$VV3W-Ut$Bh7~*6XuohuV{<~j}mcrYp%9a99kq~$Vn@2{-z{d%MQ5O<)oe?9< zg!$a^+4&5A{1KXwPEtd(lGpb-zNq|`9#p%yH#*6HFhF<|1`xn|w#tqQL1#&+Lo~A$ zO`!6+fT^R}#C+D#$et#K*v)hhZf!doKPNDtes(qZ&L@``6HC@Tz?Eb3kLiw%j^^R+ zJUo|l`APm!zt#_Wc=8n~uS4}cIRsAN$jD&4`741&yUG+#{|nAL?I&1c8+%W#reJdF z*|xn?9i*U!2RcHT1*=`I!i=H?xZsYfeb<+IQ?4ORmdRhxsv?NY#f#j@P&6ipr}Xsi zYdx3SsFN7mp_Ouf#F!mM6YO+?>=|C_0NT75Q5IxGPE}fC|$nKDx%c!WZ zw*%E{f86rlFT8uSdML;x%!eVDntt2D7{#GyQTTPur?BG5Z+^YyIjcaA(HU6Un97tR z*i-DDoYWJjN3u}iWj~sQpjVm3q{*oLAH98dtc2H{pO_j@8a+=V5W1#@yp}`~a>)`+ z(dKynSB&M2{N&kSU~ws*F{)zg&|UmSJnSH{z=A=8T|0iPY@A2D$oY>FiRKRmeh(vn z5ga#xq)7UI!&o}G5DNqRPY;^EvZicz_EF|GsL@aH5Z>*vY?(5whgk%X#WIRjvmg4} z?59uN`m8S_4bS<*@+1F)uAmQ0EcYZJIQ0i6qQ1U9HjNB4MAir|i)>HV?J5M`>FIP+ z0huj2yaAyj!3sVffbUiysRf6z14xyFr!NSJqK;%`UztsRtR3@k3ps|Z9?r|hpU4Gk zS-mGuSo=g;^Q~xpi&i*u`=8^T;-~3)o8(A9jq=9IR0xEJ6;Jx_;1N<05=PoKF+Gi> z%&BmOvLDhW9t{l*;*zB`n+q#rOu@)Y*!xr%(Nb1U!2+ReC}A_fc_~r!`G@X0Ay^kd zs&6-5#@sIo?3dSYc?EODAP3T}aiC8Yw0jMyAs+>P7#{&vK>LlY&=+-Yu2eq{NH3!O zh-uVOIJ=S&|J>1Qc*XMTtT5W6Uu|@1ZuCo-4lR3>-4kOb#us@svUc_|5s{9fFViB8 z0=KdvlQu7za1z2MCZwhg1$t&jq-F!yUwvP9MgT}ZKfk%<hOxW`(?g;voTj;#$Q-i zXq;ed7#T*|_i9>hb-B{9r5U@=u&vxo(O z+k0<#A(UE#*h+y|_H@hVyanjMCD0PL>>^84&PWcnW*Od#IUigCqVUDRAUtM;!Tjtp;a|w9*-^9BnakcSP)(B7JfzQU%Z5{{w`iT_YxrUU^Zb$-wFesu@vGXZELASkYfxI6DKM zBD1*zLDij_J?7ING=B<9@8Ul7pbpY&U_EteZANvU`)%v)WYBVp(!1({TVlS#(Gy8n zYnXB`xxt@W<-%Ip3_!S0x4VD6F>+!Pe?h4jRskN$)^XFq)DgKLv6s{V1;r^L*4AhII zRW*;XImN4K`=QPb;XLK?)~}z^W>obEAw%9O(_jjne+v-eKr2a;Hp;s50*$& zZ!JL*dQY^4m6e-nQ$tOSwzAP6E!CZa_Z~mRN&*GC(c)ev{Jr#4bd2u1m{`W*M?_EQ zzMi>6THH}zDYdc}vzn!|z0(Umje?Jxa`~5CRA}hBj0|gN*2&i#QV%b$8 z#LJ4b>>|MckhWmbdKqi80R4yAKwelSTCmeiF8YYaBqOjTh=6K^1O)6ACf{yoJN%Z@ zcl<6RN}Kju(x0#Su0zW*15b@yGtIvIah_V)WTS-vbze9*fPis?A}92d_abP{rasG0 zu;e`VWujO$r)IP=Wj!AOxLRLe2LeIrhd>)}4*_DshyA<9@>uIPNXf|J^Az+!Lpg*} z7M-R>g{hD{#jRa(n>nj9;f7Ii<2Cqbs@!w201K*Xb05r5KDvqb>Ly622DOjBUS!)} z@KgFZXzOyGzOHx1)2#7|4HNhL2`>R2kp>F@#tO$r|&W#0dYADKp92dq8$iMC8JJ z1Ejz=W9Kf6jwzEkGb? zSgTdHh1FAUe4&fu%-%H@xVP-y;7dfWdO){u)qek);ttY(QLVT z^=g*&d#1H$I}m=fmzeL!p1Rnx|C-Qg?&!gMPBS81ab~8>B}G;)^%H z6f)hLH!2g zB$&-+hoCMfUn#JeCl948lX^;@=8h$mUrrV2&5{=V8eyqTA4O!K4Mg0#w^)RQwfCfS z>s^XAFrjmZefU51?TGkzps?HxeqY)`<5xmouong&%4ihc79o>_CP~eAsM2Qi>Z--J zx7PvOZiBq2axs#&Xz9DF?PbBiBf5s7XY*MQ?O>{JZgU~5|Km6tjK5$;=<5uHqxm3H zQFW+NR`vjt)O?jY32Mf_>)ahezoMU~zD2n_Pns*kS$%wX`X;}ma@ltbPC_6Gg01Bk zGQ*vJjg+YtTJ}Ffg(HL*Q}S}0m;Jk&B~M6ktwm@>%AE>D0Z^G^!h@faQ)W8(fUC9^ zrTQl`_jQM^MdCkl9+~b%7*DiqFmRHAOGQhk5GZP;nLt_Q1xtPMU0=bi&-V$pZwcJA zaqM2C{3Rxp7SH)G;@&Hz`ApZqmiG8>=dk*5IPsK9miT5N2!Rnk4p5OaU^SG$8PIvg zphyQck3YN>$p{f779H)?;o^yagl|3bTt?5UXD>gxEjZAI*(xp+?C;F4FuaI>0*;IA zc6XYX5Mr?EbAC2TNNojKS8Ax($cTFoirV}3aIt)j6TjT4n%X~n#@nv0yho(Q`QS<+ z3e}|g*L+@DQl&-{gawhI`~{j1I7Qn>tLdyb;j+-?4mUvA{}%BBX^d47^KO}NdgC~> zI5Su=O?*>C|85laUVkb2pbH66)GJJRWWJwf(4GI_pW1F$yn4&1{p&Kx+*Mtarja<~ zANyB$j>#(=KD_eY%x#&Od5I3~1?HOmrCTpn(m>&~aq*7+VEGmBq7lc?X;0Y1|M!b3J z5U?J0Q2|f{0nYm0rwL3Bhnxj_FBBeTj__b?Al>HY~yGLB5Ock*_ld`iBV=qWNG(rXWy02ulmE zBa0G+?B8BsT4cZ8o~z)d9`NjMiQ{KxgRL2F`vq?yzekmn;TM7s?bnhX@0!xg}eSOk3i{`-JhiCoIrsBRjkBl6CTa;P~acr-DJ|j21VRq5c%dTKWet1+I4mEBj zn57^L*JMG5x!Zf?fLB@@);?RlfG+rAkOa{4wl_G!wY^b%;HGp$L#c;z_vEGP^>{0e z^ze(Q=+^f<;%YyFrmn{AHv<^GXAitkN@v0f2LuO~vmn>ZkPs5p3avB2Z1Ltxs(E}i z5(7gOPEt+s>_&r|C4=I^Vs&8C1qkU^D!R7TR%BNz35)V)coZ724r>pv%C9olXd`rN zFil{S)|HaVz_@zSKS4*$BZ603C<$bah=&nL5ol_N=oGBCfTQ#w9nSnw;rKDme!6I= zL6aPjmlk=IXEh67J;Z#~f^=2Na4QI#A@6PG(vx#40;RSi7 z1~}1KVn%L~S9f2xz1uIJUs?9Y&ig4GQJ75~X zhEwgO7?-i> zRdtbtVx zXfyrb^~rzNJ~82TcYJ%kWgaj9(CfBLPaCMjytMP&U3-7I5CTWO6)-iM0U<<0U|>Tk zuM)GT6gC8}iWc`(VD^bR9T9r}-SITXVJD!ydZ)?3r&V~W6;qPC|7^Ncc@j8DVBBj3 zW+wT6sVLY4v!%^dOn~#^cDPLsz;&@-=JvM3Fc*)>J0|H^{wOLKtwP&#nKlWO8j3Q$ zOw#dTVR#Ji~(c&+kkX!asMb z@~G1>Q75?r!My7N=+aohL7E9W$4dhsR4ej(3;k0D46Y`sc;&Zn7>_-kY}jjigtrG-$lf<=aNXD{iy#V33fSE0mEyDBCq>rxgkbE5hKXqg zp6fL}=S{kT0HAgKHOv7(`rCFrSIWS%1rQ~Fn@$Jcfacj*ss4%!lcmzq(hUoI&3o|b zZm^beaif3h87=>@O?&7$%Tv?3{gUC9526z*N^iozCQ2m%KNYPb@s)) z{l0g#MYp7U^eY|)hRf+M_LJys3_w;yNJ8QdjxYI-UtUxU{xW|aRB5(k^qL13gR3QM?u_n*Fm%${Z z18$0X_TI>VD=R4}sfBh>RZ}eJP8wr(e@#bYges=?bps5@jA@!G-~np~#=hVAm+3vR`&M>-yE zZi%)AEIycEBKg7ht=x)I_htoy87c6U@Gl^#)y^oHGhA%!J0Ke${#2Rc0w&Wsa$jN%xC};b)<9y9jgqw34rl-zsekikGHReyBVRq4O?Fc zlIM3n;2tU*ip2BDGvoC1&U*EX9RUz6t0@VB+VJ{OKBaoZzQT2U<*$(dlVp zEy+;mKXbqG=m#BCIq26@a1nkz~zg;{Ijf?=KijJKxnruSBjCdQMWGgs!xPoR&`(C?3K$2a6 zP{ro?MNdLxfdytQX(CSd_O|*lEThe@0O9!0$Eo9M_pA4td4D+YhkMGYaFXa#d(UM5 zcJE+^-dETHj^X;U6-U{zT#t6}apYQPPH{%hbVdd~gJug|BtMKb8N?8Mzg zzg3aenwgf*JpSL${nv5BkrqG~FFGcM1S)vkPfB>2&^Z{nRzd=t_{RCEWx0pIvAu`Z zghwr1D=wl;F1=3T_r2P4+Q;y~gZZ?5narqE(nLuG1qA~iyFd`u(wTjm3#(C!>#e|R zU=^VQ)Oz0ZI{WcOXZ7W?m&+)Vo=*~T@#+Lm+Ajt?vCCj5vkSg788ET5KBeQP_g9X2 zq-dT5zGGZQhqAHH|24#-;I9CBh;n||MgybyQv2>Nc?Heb6onrHkr-i<*%*aZroeLh8x&xGKG+GMv_vqFL$e0eeI+ zfw=WrHQCwOA#M8J@mglY_OJS_40sE3v;~MX0xNeqt!>8|DGZ~~)jR7Lc-m$0KG%k3D3l9@ zbM5yfYtzqiF(&p~#uuesMup-|+7MDb{AXB+3e#5@f-iuE6&FHzW)9TYN^fQ=_?)sNx-ziscOBRSzKIf>Oivlrp_lHZ-Og53; z^2tyI&;wUhyZUL`MgipN65v7rIV6@nwbtv(pex>ItwzR;_F=6-4m47k<%}bws41x` zy_kN%_zM|PuF0CwK{~?QYS;X?J!}!m}+e4gdM9XwZ>t7rO zmnr{@iJEvND0Kcxi;)X}7628rmrJXwL7&e5Qb7UM0eYYZ*bY{jxls@12EZYoEi-5H zZKEyMubJ1MC1){%%Y2fn>UCprjvEiyyGaZFmC(p^Uh-`B{_c-a0$2sz0ci^ihYjQQ z^$9(rMsXNQ;J z_m{SAb3ral>GCKomNNIM58BrDEH&U+LT5ng<=;{Fc_J@2*+j6;f;;duMq>^hCXp zTM1zl#K3Lk2Q9R~eiWLlt^>^F7=MCy|CyN2-&0^EV`5+9vE5K1A!@98Gpc5~vEL*- zN9Y&b$E+dRju|fMx$FGowPYIWO%NUcjALoHLE23&2i)IVr%lMRazccS3s^4@=f#oI z66vNJ2gaPTfG8~&eEnc*=x#GK$0 z&~b5`R=)Bg9!Q4cQx@kfVK(&Nb(UoxJgw#0P#5XO`sc4Lb^R1s>r;jSPl=DNa94nM zAWPLc0_u6w#H}Ac-oQSe6a;_*L{KwVLUlcbZk?*<+2bL z%IM$|zyhxs1|R)^6C+3w?BWKG!6cKv(4$95c)B2mcr9DP%?tDX-?b1(FZ6x=*bX&; ze)H~8y~Egp_1u9eZ^V?4M)nPIyPqw|^V`o%5u!C7t)ze>>RXe);9D93dbtZk@J3wK zNzg0C?M^9tv1gLEQ{za1e>wE}cl+w5luZBSi)JBqI7$4r^;ii#EcAT|SITJwEV4*|MH3MkMopEK*7)$iD!Yj zLleWDc^R4;fY`FzSVDqB>z@5EQl6f`JRifnz4aJ@q=n$Jd`J zywtDAkihbr*xKFj zQv)LwUQZk)A+`4Ln1ea$RsSzM%S3ZdXv>+@-&eyN-7@t_$V3sM&g|_{*qk0lN%*i` zw9f5!26i+*EP{G6SyhA$jw8yOtw$1TqQ#U=J4OL0C#=@VYO&Wa+$!4!)FLiUV8(8- zv~H7EEhTOhbUPevu9`r^ZiOd=fG^!&XbxxF34WZI)op=@im4%Pj7z#@U#eWgY{QppggG8F=fT3eZf4b&!Y|wH_L34$cOnl<2X1RQ6m zj1zp%cnU-XKuw4&)B#Wc6_qax)H5Leap_3bf8h52*xWh2Yq z(rG~!%MQq6hPRsiVn@WI6HQ1r<~#V*!oFb9)I2^;0WM{&*NnVgmXws3_xu+F5CSwt z9Y6p9Iv%2=dteedvYGs*C@3K>{ZBm$2M6(@PGc*X%88|nA(A_giN*bb*fqJE-#&Hp8QDwq3J!+j+@jb=YTV}UK^g!}Y`_l-YN`6vq2>NgAN^f$NO zxyJFj6%2TCxj+Cb`}xw883hoD7!Msk*k$z|5CRTuPp}TiaEEbU5YW-U6a~-K)m5Mf zI0O*(F#+VrX}2T(f1wDKlKns-&|(yD+cwt1xhq>tM!l#105kV`=5S>TPqX+BZPuyp z1}%Y@2_}nH{y*p2jPb5h);(%ny*}D9(H?kL6<(?M=~f=lzd`O+aQutR=C582*e!G2 zGSvF91MeAA$^rPY4a=%t7U4$V1SmR&jT9MwMB;iyH;K>Jr~;CJg-;_M4ASV_mU{Yb zuPkIQ&3m7R^|mCL9Vrp7>82zxEcF+L`Sn_x`&I=p_GyA|)P-6gg$N=CivD{S%Re|P zi#GZpYRpJhdmgOY1EbFR{-H(U!nU@y_s|Y>C1X9zQ@;eX2H-6CC=G-J{4an&l-;y1 zpFFH1L8Y!+JeaBcP6&INJHVDGJ5p-bDkI_9ykIBkdhk5$SpclV(D@4A8S9ulhAf}K zyP6UWoJ2F|L*xm{(x5e1fFXyVD3IY!HY|bN6x|zx@Bx%6Y!!P#U`#3DYRl&1b1pjw z|1ZP4y}kWI0&GBKlg>J2iq96Yoj57jaANA+D17c~^ngp%LqfKy(@nS?6}`pj(g=Y= zvxMds*2>e9J;}xOW8%Xj!RxRM5aIiSOd?qrrv`?^XB8l~#6l~xRC6CX27oHxm%a!c z4b6C|GwKj@TXTzxX3j~S^0n)isi~>q>Q*Q<2B~Z2K@u&%*@P(5CcUR^+$4fD!@JziWM8+ zPw0PnPBB>TY7d*^1a&T~ngKF@?|08B?dt%$VRs0?3w(`TcllRBNMBEJ`H6jWb?7^x zp4$AWE;58&+BfdlxlCzFh|1(fzWnn+hi!U7Xdv|V|HoDRFVgUAP|9Y6t6`^43@rXz z*ImznGsLA`GFPjztt_qS{4HI|i21Sn*sh8nMZPx-hyyp2vIHUqlP5s`VBWwO#sv@8 zAhRbVpob-nQ4W8Puf4BNwejm+Vztf?+rAf_T-x#Z1d+C;4Rh~sf@Vpi^q>Cv&Me#s=SEZo)j^Llh~dV+RGe_6ze>_Rc}Z(n06?~X(^d$kP&OT;7)(1zbI zn#%+N9tUG;!_qADp_(E1n$}Mra}al~Ig1L+6{M%9&p$uhHqozhRygxS)(YOOrtXBk zjrsTlVK_QV!vd(K*+13551`k{6-*B{d@F7R{W`_2U_UTN(dMhprPypWqo;py{~&g7 zQI?FSfbbv1u9JIrZv@QOACL&;7e(!k{fYKb@}a|G6PJ*Pif7t}R@|WM5v&$Lv}Swu zz?e9hcqfg@Wl;B^GGz$%&7y&oTu5>o;sN+vGA5ZljK4O9vJ~jfj7CjfR>yCH`85n>;M=me&;&3)}ZZxB&K_gbfaokm@Uk-->B%VIjjhWS}mG zjX9d>PVzBxcG#;}hZ$g`Ym`4$8Opyem(BVK;x*E;Ur;#O3 z#HCvyFEDxn+d&7`(}*mT`PTk`gXHk1o;c2gwU>Sc7Y0s1J!A#e<0ZG$`1)s8bDPa*&lvo{LZbSmdec!#jTVu)N7nB^Z_)&WUY zsrre)5!{JGH@l3hpJi5C4YCYxwiM85B6| zyB@M)QDl81N*RVP(eU7Nk7YRX-v2fI2AS2aonEk_VJ#(iXaw(8c^*8?97O!+lg}H1 zvv%L|TSa>iML7=a?gAGt5BMw_mP|}HkskB`AIW-a5;0bwc`BpAGWYmKqBwLy;8#qW zRisz_;jbuI_&{J2gkre!@>J6cD)0zv1*MXfU{^qv&h(BSw52eYprBLG-Gm8Rmax%V zIx3|E6L6Eu5bJJ-N~LS3x$mRNhC1f=S0lkrhjq zsW#-4=fy6Pfg^`)B9~U4AlvMKq*vxXprAKk-a}Ugfmfm1Q(;x=#j$g$K+U>;76~;}AaKak!X&Z;s-wKIM4m0F_Yi8d<3~aP@s5~Jo~#qz z9dL-9nW~3EQr6u4QuHn_aMZ~`Uh-P%@1YD{H%FF^0Z?jqO}?O2;(}`cF|L5^n7Tz^ ztgactvzLw4$?7teg;ANNf8NHP2M9U~tN{MAeI^ro38^rVEdgh5%s-o}fK!HgnYqFn zia{9V26tW^*zLP_Ob4WThVB|fsR!m;D#OGI_eCj`bxypHK@)Up?J($p-pHsXbV~aL zXoHdYQ1(lp1b;wL;CpZng5+fw*T5fk!XH9zPp2YRC>Mi%IX7XB)bw!0<6v)Bt(vQ}|N70F7N~1@VI4ep!J;ERpzwUx1PuOd;`)|TG4C*88tc0s;}IotC%AEuH> zx4I6stl}L%dO10{mM!qn)JRwBx}yUKNc=VUTnmMDU^S5|Xl4S*pNNX80K|wE7Yt+&p;4vL0BsW7K}@0jKvFycS=?X;;2W~64D^3caruGUP{0{N zNDUs~xV-_F@p=c82H_C0Od$;-g>EAg=yahAMFpr|8d^StQw-(2#eaw)0wqv8b*Rzv z$w*}a3{b%3{AXaONbed<{VbgxA$Pq3KN9dp;|(q1e$Gf+XQ{JY65tDK*=fKUWMinr zSH>rDJP%}$KgDeplxTk#<){VQQ;Ye`pk&7T_3M{aqfQ2wTJ-w|AN47q?m^sB!7yxC z4gLj#;v$L_tT;}Td{NgO%6t~e58?pX$tpV_FqkE>plU!b#Mb{z);xI$2b2p40&1cQ zf(RfHx(m~*7$EP$+L3a!y(j!MHvJlhjCZWCebb^KhD8F*7d}IUhLq`34elgRSqrs2 z4q=l%_HsX@tE}wr?+<(y_01@R`2Q*G%fqQ|+jgH)Nt93_Q&`5x)PT@HGEX5xnWc=O z%*vB8lqqu&357(2Oi`IFrIdLpQzTQ!nD+VeyzlE-=+WNsk<3hCfR!u5=Jb?gH zAj6i2dyYW?9=XCu_KBbK^ELP{^k{D0Jzyzc<*ZnMCFrv#YIJ z^KEY17b9%E2lgzwmN`9T?wUOI5ehUvy-%X8rT=b{G=Bu<)}(V0x=Z38>`vmEehd56 z5X9gYTZzXcboF5ej`RR%3_g?M1mP9rsnMl0s$MWockbSuzP#6W@oQj!C(248IpRaA z=9~ZxBJ#dIbC2HWyw{-|CSvPaKLnM;>ne6bPKbM6hci#7(v!{`7+D_C8C4ta~1L@(-_&t|!Vv|VUu=>5qrU;H3jKP^eTgQ1^V zXZ0ka(>g1g(j9_nL#XZ08rg8*d}*|OCvE~-bOf}2%RBR4?|OThq=t<=~Q z=ezspPl+O$9}@kB4sNS~I?ABGU%h%|{(zX^!cY({d|Kk^UtgUG^!hHs_MIV2TRu7x zT>7ubB&O%)6cA1g!Lo*~4nN!kO|LESfMFHO|N2sU4jX6zPj4h*{x9ni{jP?6i7uZ6 z3)Q;R?Qsb6Fqzm&qtf#q_Q<*qc?j2>DU*iiISqWWrR7oXX(CF5BeC%QLM5VLB4%b;y3zjTHP%SbOq-dXKZP0h&z17>@~tJrRj9x$|JdHb zrU~Wp;~S6ge6sqd5bUmv01@cS7Z(;4)e6<*X0Ej|EE!W@9x`3&Uay6^>4SLGS1&`# zcm9m+<00#uE1$&I!dBDH^400TkGg3s5PS?%thy5T{+rO8)^U~x6KEs*`116t z?p{8}?j_~I8IiWGXN9C%@8r&BLK8oJd?uf(JIp=kNtmSn%MOKvLaW;FwQPFNnS|R8 z`lGZh*@hA^nT~|Eqt!MDe|&%PnVtvD(U^#}ybD+m(neNiJZ zWvBOl3Cs_1?NgQ;dXg5Ai(E`PR!B7|F4pgVU!B~OJ|3y`lTeU{G;t$;EMkq-yFvJZ zpy?^~+UVI`q5Tv-jiHy{UU2K`=RKDz*1~aWJSs9;|C8APdQsa$>~j|4?6HOOf82Yc zX6+EiGE6Jw$dTXDAzhlV&Q4w*%5G>n6YIv?#;WnbY@g8Z)e>xe_wP2@>D_mltS z((?Dws7SF0I9Z!5o?pYpyvt9K1bW+Zb2)2A9fxwqUg?-6QBJ8qtIJ=bH<}Ugs?i`@ z!mfWO^MX=~Q?5~=%Dk-jV*)kuDD_<_GVg*Do~~BRbLDe)_6Y0h3i<{g)JY&DrFLIb zHgaz4fedIc^35-3TNXpIti(NV{aRPTZaV_Hl6}YX6RrD{4@*0k8K@&Pd~PNtC;rL; z31qZAmjhQ3L6kW3&>mxYG~{m>5dzwEc9tZs2=q7)I^wPNn)!py5gkpM@9C6lwNM9m zczWLRb|?M|e<;WjW2haze?z+4x?ez;z|1&^QFC-&=@DA={K+aO13@Gvk&TYvM!jk& zE$_bM`yK`+db|o}-2NhxH!gCN07V)y zjLNxu1ddu$7~Q-bPP1IYWWnb<;<+IGTHsC;WR}M)PwP^O`8whey+kku6mqk`Mh%#G zhErR=uBQZ(D(mg^+P1X~ku+C~zB^Kdkpz%ej8lJlNw{5Uk5v5?oKI4w5K-TsUTK65 zsNTSKb*ND^l{yh;*h#Rvun zfACj*9J>O(O62kawr|s`qr@6T2S48uER-0~c)VDX# zr|+k_O^o|>bQqwAbSIsF;uZGX!fKw!NMH)MbO42qgO_c%j33NhpXjCXQ2Vx*a+;Qt zj^svVky&`7%hJ@zuHW7EY`f0YDPN&e+TPm6g|P)0k7B#LL>SZW2)65gu5ik$+heRS z-TUoNpOmW&+3I9V$h|*Po&inwq9neR&svX^&-(Uz2lZ@YAgPEW{ntiqtB(d!QIY)$p(2m@mZVannOfzYryA=7BCU!QI_+d~F9sZae4Vo4Q1 zBQ3+rEHpda~hkQHF0*y<=NQ3Sy(vz;d0_Z2lX&Bki2+CnT=jt2$gmj3fi_~ zhdz=gU4b)ext1;c{icq@(kSFo@5oC%dt*sKk(MsUc^5NtLlBYge6+hGs}%i(Jrzb| zSv1Vz@o{?3jPy`%X5fPu2or<+TwcARC%EFy!<53Ir&ycN-X#VcmljuZm(nyI7@_bB zzq?*LwzCIz+$EzX&pII!QeqI}!7Z92R6>qpb~t$(hh z04kl0W=ZdWX$+Ohp^>8pJjWKvcO%1tW7kcHQuRCLi$|<8Gy(;$BGku3{--}Aqyo`} zSHJqBLmQ?hXAayTz@d47A_AH$DuF{=l?ftVHhYyA5whq!1$mVYyiH8N$l+5AZD$cf z8wl>1gl6XOyM(hCnlSK;1nf}1y@s*niDq(7*}6H)+!OQNM;VcBt9WTfFfofZWrH_> zPL{bK7uU&KH*aRN=h-jd4Rs+O-^5eu?2w8vRHYlcQ1cR!T6p0dXd~|sXaUijj1$$( z&^obp%wzhYS1+cnvrq{>h2V1rAQB_lSRU0qyz6<;(A5tAp+|&Z=$9{MWy7`kPtWGo zE|oz=PZvBceKNwhrH0OB)g7Y(Cq?K0VHg4J6aXS@)fv~``@*LAGjs!nuUiQ9Zs;ai zjyZPlGOM~*9Ue2x3sJA0y@bWM9TujKVCO~-wRQB6IevC=F@oJ2FcO{8^`I+P}lF( zZzZG{Q?idV`gDsonhnkruPcQ(_IBk^W+F}sB83Zh!E@G9QA7uclBZ6xQeIP{l!Rr2 zLFwiq<=U3s=j!6IlK~DbFD#gunBp&XTih7pqDe1v8NOYW;_he1UBd(NxQpv1`JNDS z$A%5J_&!8LxpK&CCf^$#JLlp3{YP0qPEctFT{qI*gL)@G+v_fWvVF zOMws@qb5g9OM675Dx!?AT}LaZJ-?cmp-09%xbr9@C)Hb-M=1PN^hyaM5|nkBa8Sv> z%y3yLGjUY7TrAZh=)WX5RcNCK&?2U6Ftn$KK3w$pR=r#k*)bDv6!2R9Bzi*SujsiX zaqHVX#4!%4pS`?;IFUFjR1E)IWn2ur>zfC&2x<}QzZ9evzwUK|IEgd4{u0N*T7Ox+ z-LxJMg6eNl{g*f57^GZw#j&pcL$p3SdWQoVFwu7uIdib=w_-H?`!#5?;j&c3N$V{@9A!6kggzhr3dvMWP;Q*F23F24@ z+o-6tv8x(lFL9s1h57)jX0!Ys{!h$=*7x6kdPH9OLY5X5B&jU8yEn#*4eob9JZjh^ zat`@K%+cfWkV{gZ?48~k!t)>7_n$v+4z|8`_J@@Cb0tEC^8d7I86`I9GyUTnQSj+vk{nn^Ow#l3`O8X)2` z7KRyGoQPaPf#Za=oDyb;-Uz%zRzQ zwA>>Fm{clSX*|i5hWS?7Nays4O8HxjPEM5 zWfv`B+T3uJHfcdzYy2a=ltJOJEe1!x+CAuTTc zH9y~q0_{b2*vq1Rlr-Z9FSU$5e=aKP&~@;VDoBhUfs=^mR!K^`kG1eT5VKT#0HTg8 zQu#~A0C&Yy*5O(Zg~T45E35gjkfo@=f6k)doM=YPT%_;o89&j9j>jvL6f~N8j-_ z;fY{}mKIbT!?Tp*FmI@tbS`RGwCqHKcBy;y`3VbT(u?ydkakRMdYLw%?!aNL)cb`ik z{xo8}c$x4q@Zm=iqUL=E*DryYj)URY!J~wGt{uLV(t?7G?Kv0 z^LT?7V@qF*dczq0mx2ngV>Sf|sdv;>X)QWlSWdV#C4J$lmEN1iZB*^dDARftW zD(>mR`jO%+c<#PXKn$d!@3vaaqoJmbV%jKoefrv1(39#&*NE~aY;;`!QPw~5(jl8% z{|9=@YL8`IO&7VoqEHotu*Zm450P>p{d{4}=&TU3eUW*vPV8`xo9v*o=ECZdYb)3Y ztuJge^&TlumDNE7Ejvlp`vPM#&PTnZO}B@aqiPLZBdnEl5xR}Gpb^<(8cw{jmaa(o8&Gl)XUr9s6TP`@G&Ir8 zDM$J|0?=$5HzE6oP7eW=|(=&?$mzcSo94h&R{7Cc+SwKX?+AxT86(r?$G{B%=qZs+`O43;uk%^=U5A?uK*#0Mb zDSCh5D2MW8C3x;YV`X@J-0meSlsD_f{(yFs0)jW6Q#N8{-fe9k=jy>e_9-II1EQS; z;emRVOzxSQPvVMwdo^|KPsS8tQX%1KTOkB8WPO))5%M&%o{NR$G=S^_RHT#`l}|kG z2-rj&r$dnM(G8x#MVH6{Uop;Zu_3loleh;(rB%{&$ zzluC84GcUyikl{m$1Nk{911uhcP@P6u~cf=3XiVZK9=rW=W|R&W(`dGt{?=|e%!qH zi%ot!WT@WH>BL9*K(Q(Na1lPl28Mj-Nq&8Zz#l6zE}GW1$Ks}EQast5N|~Wz;)uni z>c)h=W6saGt2b1QhcqXng_6)O;oPrupp(~hQlfs5^{ve^BA1NdddTB_HQ0T|=Y_>3 zMB0(MS8N(zWTR z^l`;W$H9G0znS_{-uK!$QF`kq4&=v1-+Rkmex7_OwEq=MJA#E(#Enfh3FnM&7((%# zn(Kd4rxEp7VOdJ`kpq)~$=XGGE}t+sM>z(D!ASofm=uzd&|dr8*LMvWibx$oKjqV5 zkR9WSyZ}s>G)qdigRWZAF^Qt~S?m$JHnm0{iLhM#Et2aY{_&Fd|>0p z;_KLQhH(xpo|cH5Mr{P+0um+#FyD$x-PZmK~NMV%0SD_P8RJ~f85 zw{;pCJ(;(-ud2oa?pwl6x4{%^$DECe?+)X>u4~T2E(HKPy6Ug;?xgH_S^k=PaI|Z_ z`0~VdthaM@%~iTOLubS}`#(N_RxHI#9 zv%WnP#(LiUF+eYR)xo!Q#iIE<_PCW+k}sGG+#1-EKO4`(PKqz-o+~9Q9Ic+SUn5=W0qB4U(m$CG%;% z50e94rq#+eIu+~bO>nw1JxBX9EUsHO4J~c^soT5m8Vp!5Ol@Cm!*T09&2`J^w=<{X zoXi81|K2VMR}P)9NC-JbT#Bt>%Y+)fvzu^upE1&P?95$mdD9v|cdLl~C@vgy|8y z{(Cj#f43MN+5#HMQ|zj2;08~0(v(ub#^Wag{ZB)>pT4@fYPE#wDEerI;fu#TK_+|{ zT8XGk1$9fh1Y!j}OelXJkuJ}TYR_Y!0W(o90?_!40duDnWM)Ih{f7_hBlj}uEpa`q zo-B7B4oe0EdV_AGF+l?&DluOqWBII9yGp_VnPh>)(vkQ9-cRGrd?oIFLG_;q1)f5u zeiQ5WiH3yv2PmSZ6(vc>u2+B#8e;?&t!-a0MtUn?{aZrW<=XH0vqZ;c){^LUA?Fq zg?i24R$!np&*_LmcKV0V9#GZ+e?o{%K+AAF<2+^=&o*+JY1SL8aV|3Aya+ij#~g8- zXkrkk%hJjU`7n{!&-*Yf-|8ov^1eec5G6)bmQi8W(hRVQ3^*(|?v$HzLrtauK1&B1 zLs&Yx1NkN>H9ljE?|c|#677gA&wyj$uihbc=s6S3Pu8#Z)`c|`3uYgX~!*@p7OsZ?swc7qp z$aS^?XnWRH<0G3LUP)JeZPD8$PRd7NsIS8xSbosjeD3vzU}>-0teMD~WGW!^R1(gh z(sbs&sQHcMFaESc)pUIK{i2*ne3rE^+|nfpOn=fmmWgbQ+(6O@gEnm@Mu9V+x+L5{ z9m%AsRE|`g)SOZnYfaz+z?veYG?>~LnVB=O+B_@n5J+&%-5s#mg2VaI+RMt{>PQVV z4qa+dpchOGDciULAWcQ;L|w2**dHYMSzx`7Ft{fU4UqQA)1%ia(!3c(U>1r@$aJdG zb2JUVJtt&8YyGvFy>1%lD!ztSCQ<*h$@v(JTz?>aRppuSOO3WFpFCo&7`@#_JHNC6 zl;0x?yD9^RApsvf!IUdyamz%U#H6GPwS5s~dX4YU0v6#Llf2}jX>e?6(WHFk-Q$P(fryVn0i3Cc#w)Cy#+Rjwe zx@0c{%9RiWTgYJM%Vzs;+8$h%x1-4a*D<0B=Ror>dcvUS^l`469RUDqFnp-%&|9j9 zfiS%_xt-j{k2--)UHurK9eo0J|NY7<`?y?GAawKy>Rasjw7u}{dooTeWLp`gdwYo9 z_wbr~zpqVUA1XoXC1lL+$f(s*sCNsdp%&PGj0FOx+1MVr7-Djm&Bsn}n1a1dpKTf9 z-UrMSwcruHG`a?IKy-4kGg)M>w&NgUz=g+mCV+kK;jLB>-rU7^?L>LUm%$j8MR-mR6M#L z9(b+NxPIdtLRqC+%4u>%VgWzZ+8QH;Xu}CU)NcxyFq_cOBgj=rW60&Cd|;9z6X`|9+6G4c|%`u;^obxzq0-N zs0J6Dm)7cb!9i+egcv`J&O(vfBXn7J50(z5d&IRWY{)*q#&Pr$wfj zH(JMOO4Rr{I5@^q#P+QfhqqJ6Ptm)uI=|ZCEEdM{>U_rY(XB}=b&HS-G?qQ$bx7>T zI}b-1?dtx`UXSd*JxE#jkpBQ3m3G?)sOKB=m|s16-*wPy>OY&(mEgOJOu;Qrd3D0}No}-|zZGF-R&jSj)vW!?zpFPjL~z%B;MZ-V=$@Z0 z);4>)yQn$*s}IwKfUo|=QWm>1Udx$X3w0KEcs*SecWV%q8O6P`=>^*ITUij34X-w)jN)oj*k5~KxM8}n*hZA>`2dih}__a?)GM)}8 z(d%%Ut4}$y=@nDF#0!g>ZDT0}0?1nXo_?g};mhr^S_T}mS41l>09?{}{#CEUi+FVJ z2ZCJYIy9RyQmF)$GQ86rp&f zTb%NcQtCXh;%?B^*vNgDQ~<)C|qSOw*-GbFS+#M)>pfk zH=<%b)Zv`Eih4&WCRtX_gLS2R;quAwEc-3Gtllv$4LAZ{-^!2@C66B6qU$1`@5jE~ zUt6(v&%h1#9c14cnZ4aD!|~p?Zo4R1Ii%RK(=!at=UdNpF z7a$=SJhPFZ>|*vx{qSDM(w|*}hO_!y+Rc+?Z>|xV*5x!ScT_?Z6lTn6p-95@*5?7v ze#6TNNlwL1+qusV+qd$(5WDM(4)usr8^bofj(A3Y@qYe z-?*kyQt-s(nZ}#mh%%Vlma#~;Q-*u~u=f)cj zw-sxNG@j=+tdV=jxQVYBqOTFdOgaHkZqd`VJC7zlOg+7E))wO!!c~JIeJAW$PHn&J z63)bB{#Go-hH|(-+k)-Y6-mRoYb*^Z`&}b|QIm3MuW`t)Xg#C|P7yoZ)H(cdRogQ`mW9N96uQ8Ie##Ev1q^Ez*8!Amb%j4Q_aIc>uq;2JL7=A z0%P;7{;Ir3!j@(Hc8<|YicZWq-Z5JIf8Fpfn`E*G`gLo7G;`knc94=mYXJmX6r4vt za0=vx>a<}Vtc@tdIMNFoc+#3)S~HKKYcUGO|3y>-0?xfqAoz`nsw-wG?5P@H`KMQr zSzkroJEUK5QM)(S^qB9@A?2s@#24qOo{<53`3dGgPtTA2DE_jPr+ymlf1Kgy-o{%J z21Wwhj$`0T=5Q{Bu*sKCEnHq|!rf*zSP zm>q5JkU2*O1Hdc@yL~(I>Q2>Y`{sKRPc))yX$qy6Wlf%*+>_PO)QLl8`vf+^!7g`rwRYgkDhCD9voBWu^s(rMn%{?bB7v9Zl z<~fLQxlte|p2GU7dTD4`zl-|f@!`|x#YIj!Iv&~g|Cg?r>%ZHUL>^4M{;y5S|89Z# e|M5ndf0jf9K4gZrtLDi6g_klW&T>wg-S{6RnWpgo literal 0 HcmV?d00001 diff --git a/evidence/track1/Aprime-p99.png b/evidence/track1/Aprime-p99.png new file mode 100644 index 0000000000000000000000000000000000000000..17ec2163a592cef2169b0ce9c5bcea8c3c7b3892 GIT binary patch literal 52909 zcmd43Ra8{(8$U`TQVL2pL#K3y)C?^tjZ)GrU8024(9$(Dh;)}qmkiw~Idu2H*?fQh zbFR+CxjcI<7dkM*-tT_*^L*+tQd3nyLOP@F_ShU;}p)6e5quKd8&rwB9HvqvQ~ISsm}py(Mh#7rLZat2~k+ zpFy8ZUmv!UBH*nrP%$yyyx_Rv_=wr_oi*YGF8ENv_ptqPS&lAA##iGaXf(@V2P@&n zK*Qm=L0Z<_-eOi-Ru=!|virV=SQdf&e+P!26FS3x2R=IJzpwf7c+iOd4x(2w|NEBm zZ_vU=Ji2Li@Y)3%rJhMe_^efLd74w}B_vH<>cNtm; zc;VD9xJio#&Y0vS)z=qn`BKHHsi~z=mhKIX3q=jce1#}CBpH8QY;>#-g9Q!_PPh1B zTS2U(l9G~V8?|)cqGjKYyV$|q4KKlB%PqfT${DGXF^&A~{Co2AD<4TKPo5?`HVLn;%rbM@#q!5#RsrxNm2XQP{TM-~W6{zv<&X1+#kQhn@UyFO{@OgNL`!l({pftIVnn zIK2>XOeWtV-?loOmKZH?svgy8gQ=bd8c(?^urugJyRs!qxMTb zNrh=U#Y+yo-(|YxswOXLtq1LM=IiYXEc#-vHZz=$dMTahG_oa{O|jg8&6ETzQTZzP z!0BMN8a+EZyR<=7ULJ#hN(8!QRQ_Q+f9IV#w{cT&mbh2G<3fY?*>-#xq}A_|`za}R zCdY1SqVC3Evb&W@QnrBP506IY6}<-%4!z+?g=&MKyJPH=jiKUnQKGr0qL+0a#(z%s z@cI>XT=*agQ-3%uiFxuH8{$3Bn0=Gq?sr)hROo0ozX}eDUvBZ-#S7&yU>4N){{4H2 zR^ij$7|QVN?QM^$Wbxuk!}`}UGQl@QgG)r%*x|5Et~mQ}4phS&sQ{hxw%S^LsscF0e;GFQMAx+xWAX+>!heqXB`(+h(nYS!E7>RQ|C-|siBlu?LP zdmq2ob!T2sPxt9KpEin!juyDe)~aPY0hNvanuJdc1U?l?n7-=)W{w?tJ~_>D9`^eX)}Z40J2kG4uL<$~hxxEBDVIO@Cn7OHA8ap1Md#NKD!! z?bGiL;)yD5P;vBy`xEjL(a?Nv$f~NUnon`Nzxf+Z%eTl4Pla$xZDevCa;&&F6FY7% zW@z`N3S=bg*V|4M>6~szFrMNsnM0ARImfpBHc1vu4s$hS{#JuY-8MM^MVq^w_O?Ae zJ?~D|2RCt?+zoj=+#{!6arbE>Z_af`z>3k=8LTUkr3O&X zK~cp{o^>j>-RVl5Lnf=K2le8+@5TG>j2l@FhJ4hmQD z{H;Zu`#+{ki^3T|Tyir~bCOsandi%XhovjbK9!5+K;`of>%7fZilf=7f4DmxvyhJC zdWTQMCM6}cVWGy@wBX7US4NA zQz}^>a&##918zLdL+MyFoBN@-ARM9l`&(ySPoCY$ht)ws?@%m!oh?n#W98=j{QOCU z>%FtBheoUs5Xc=k;tRb*$%VtahBD?_x5k3bl9Og(_16 zTHHyX2#5;z&d2@vLiJoNr|bA(JW<<0OxRt_8KqH;W#qgbU%<&aL^se@ijBp+6q#VI z9QZ=|?&z0;vjx3$?1=>@>59Sr(G9T9<(-J>$_%^jtQ$ALuAjO{i85@agG%wA6ma!H z7XwnQbG3_9A?H$ByKH?>jK~vHH(54P{2CW@yV64vT!6Anv&XOD6$$ zfp!zN(wL-Ov`m{tl95c^(<2$eQ(I6Z_kv0~CyYt@W=!HVx}`rAH?UbV-dwXo=|mFR z>YYX}9qnd3)DzwoXUsc*S?Q4TaDPkK>UF3`8hwMSd+nOMCUcQ|kJj@C6GpH9-B`9j zF-C_!jnvHlO#t?KxM|RHX42A{v30Vo+DLhx2pZ|MXHss)1v0q_Asa%>IS)97w7J-%A7bC{tkn9D=z4dKCf|H zzvEa?<$hu%-GRJV*lB{UgxA50-D(xls%1m5N>{4B;F6ZDDO}jkQ-=56t0lRoEv_e= z_m5D`6`zow_dWdm6`q)GHQ=KPLw?_FtcD(DP}vaG^?y0daPBBIjOafM!;n5PViGQ| zaiigtzo@D(8-4C}w|(g~h|I`34X?ht*{#|UXUH+=SaVYy3QSyK5+wnVsrlgb+r+QE z?KnO0ZBC(RIV9nOMAu@Uq0^D`OCmQ37K%M=-3ZBcmp_H9dz)F=QUru^_} zr6g=k50~4gBn-S17vyQm^(vL?RRWv$0mAQ-> zbWV3?s;d0Xw%@?-IP|}l>~Cyr6mkGCt;~_R>h$<{{3WX<-PFa{j*z+mH=B|*LfXDm zs}L`R1Aq@@4mM-iYK9~)Ulx{Y6@DF^n=|mbJXnHV6VgiQX+*m~v&?s!UqY zKRoPw&^zQ==+`uuIPh`>N>n}PzL>nff14N&QVzwPr;mPc}eBP$;2j+GFA(X}v zrUhPaq)zdP5|WAC_eGN@?z97lGN-Q7?MC=b>}}G@%XR=ImIRyu+ujDz`Uo_blO;E9 z*56JXhD-WgSl&n+fyj}HmX~+&X%<(mXScGk8qEg~$TZ8)x;Gdy$TOqpCJ9YQYSpVW z6w}w&H<6K%Y2h32k=*zEsYDw9OFz0$Y2Rikf;NYw=84%X^(BM_2bua2*q~Ssii0dH`A{fS2NhmtJCW^ssLacF$V3YACA#Ww1KOL^ z((XA+Z=EbOFr)uVHm3%;|BiHRUNAR5YC@h&rRVOn-JJBpU82(*lwJ;=Okck6_H%gn zE`WQATV2<}xNFaH1UE1fmjU26RWm02AhE=E?FzM8Xmr-_%6r^2p}X)P=71wmca#K= zc2RUU?g!+>uxjv`h;t{h!8fOFL17buLPkZEdEjhV_aQH3=*vbTu-BE^KXwJ9B2or0 z<1l(`XC4Glwf_G6f^V8T@QQv&BS*^d=y2ms^M`r2&gmJcxh+qdcA(e#m4EsgA0t3_ zTKun1N)HwF&_QwwVjSxXn#+AFD=VvR$dAn)2(1>YP;5fIA<*`W-$t@NpF|%-lZ|xL zEz@S}AD?)iKJcthik+ht7@H$_>ek7?*QQG?o+^!8+-4mWqfBg}U41?mb2by06ShM+ zC;7(QFE$!g16S}c=}#z`NxJ|5spfSzB*gF0addvJ+vzHb%(xN}w3`Gs=H#52@R)V9 z-Bi@H^S4COZT%b0#;&2Hv%0){SAoLj_zc#0%p}YRS{)CXUh9i1pSeU#^sU|^oWCQF z`)Cn^XaAyjiu~F!*lSds>a2xi1_xiU-s*Qmon#Ge*V??GLpTSkrt$Txx*7dq{+P6W zy_KV{_7gS`q9aTz{!talF(R@4%KK;)CMBfB#6zA2`+C*563E~5vJeuT*U+HV!>x-* zD?FT_pZcZcm^lq_2%S?7TN+l~-*7yo^fd4Us;xTx2oB63@yxwn`GVEfmuca3x&`=@ zLb6mVRp_in4JI9#XXh4>6PE-V$P}oKFM3+zSU4xEto0?E8crh06=35pHxhI<@(Pv) z?Z7iw>ea}Q_-8va$tV^ zYwlvE9HVqsWnx($7}LTUP#w!eEe4q@wG?gHlTtcFp@GtfnfR$St#mC&BIy7H%g9?; zGkCD!vc`;0C6?0mBTIlXEG-&c(K;5@7L947 z71nI{F@4Z->ndSJc>?6q@7zCKX*5+$za?vX&p7sFj|1Y(xHXxCBUh9DL}8k1P?{vx`K|sY<&8QbL&91P|clrz2e1Y}POaZUYY%4b8Ig#ivlk*CL z+R3<_I`}RT^`j=9OnL)qblD(R`mBGnES#Lcjv_^36Vm9g@FNNs9Gkvm3cwDd z@C;MQ;ikP8<=l1|)-%(Oo=C6#t(LqSJW;*a2rs#%o2cJ1E(l)AcgKAr2 za#CImW%AMYtPNJIY--F4r_l!(!<@tF7E%I9Pb@$${VbFjdfh4Xf6eAb%Q&5Qo~_=_8&KmE5m zJbUXu1vzl=V3}^NJje@JTv*3S_Sw6H`}c6{Z3t@R9~Nt9hdeoxZU103P}yZ}eh<8Y zPNA@g_obfsTmg@t4lU(%3^b^)^Q%9LWtzu*YKIcsS#H^EG)F^jS&n-Xez7sIeOoCV z3p<=&i`AFgYV6xjO$zZ+M=~qh_=!4MwGNF);W(Z3LGSLttJww`?gzXVvkG&PL1Ze{ zK?Uc1v1imO84(~yQGWM5%MIg)rps3zcvAb$nNrW;YeWOHMALNR#5~Vn)`-bc?VZsK zV@_}|>|s0^kM~AAHZ6JTmCV_00#A^&xPx*LIpQ}fi+S@dGc`GrT&4#`D(yai0VFRD zwXMq%hXqe~<+(@kF~M-0xn}`kRC1(TMO2hb;tR8rXI3dlj@5Qq5kIoTl39LNuI9-He zC?#ad5|O~#f>UY$ED_hwUV0u+uyg`#t*vn`3{&RI8TbOri$&TZ?6KG1@4D85=R~pa zme>N*E$DzXpA1yVW!*Bkjy}ms3TCsa{LGnXn38l_qY{fLda!<%Ed=H}Lr$maeiBCF z^IHr>uE>|tC-223$i5-^vYi;U4H>a^$`X%8-X<*pAlC9i*zlpMj--EYL|5BqenljJ z_ggZF)b#N3BqrTZuJo(ji4lPC!&p)r#6$x=CGJVR=4-P`zVlX|plm)Ze|QyN-kprS%9wYjQE6~5K(rpY@!dz!;1?f*)g&PYG~xafcIRa2Njnty z96l_x|D|@4ol^N_72zbGyOMr-*MOH<$BG+4(y6@lHo7owFwSc9(-XQCWpj=;k-eHaa%Y;p$iUF>p zs_1bBD>=%ByK+=)U3dJRg+X@mnn6y!X?hw3h1=`&q^;iOHQUku)fT7UU=ry7yN@cj zj!8(472&x~c#?E^r|ZqB_Mk&{Lo_N88<7O~ygEHo3vXR9J!FmbG9q;FC1E+* zgEK`SQZ(B>R;c)>8$2N@vQDzv9y)l@-mVQX%YN%q-1P;f8(QG+fK6~P2CbESCvm0u15zk&6MFUl%#SdbIA%G z#@G6*89?oodB+hjQQi3y1SG~`u*k_xw)BuPpEi|MysuE|i_1JNYV#a@01^JU@X9L> zs-Xq}kLT=LLTo|S8Ts3{`c`$4HOiySuN%@j2tVKFzqPB|m<`h6YOaps38;h$h$k8P zBEJ*{rulcmRzYXu+ft$wF)W&i%8#)aGpIXj!!Dx`4{{ zJFdPz!Kmm7=^M(@43K>;!8pG4V0!pZ#nD%rGNU;s$kdFVY*kz9ejN4jaf}N_=`t!Y zOx2OpObKldT3qX_7nee-);t^E znK;^XVwL=gDm!cunMj;ip>gmBl7NAT;FK;pVWcAk^I3=7AB1OK!EmDe3D{DBHcD$w z&i`87mr*I!fA0pi2d9!(e{ti=gcsjIaAFEqQb|87yc@ZALZSAm55tW(Wde7P-mzkq1(x7VW{(@J@GK# zpZS_#=s0KMY;6VngE4Exii#W1m3lJHqM*3%uL+c;yXufspe88oyHZ5Fh3Z9taqgfD zjLrF{<#|)m@fe6c%6J%z9ca+GNEUx+I{zER*G-~djkx(LbxmPK-Hf}Wu=HEB3gaP% zSJK8I6J30;;UM%QXLU3t}{c^icJgi*YOOVH@;aYFI~FM z`(5nmh@NXF)Xe|{`wg4uhBJt|1jdI1w5^Fkq5yc4Xqc8sk!mx?5L102Od&b*eyU7Y z+v0`eTn!V#6VUebE#xLE47A%7wLU8QIznhO`E;EWA7vcfFu;e9Jg%?mxGD49zdBx< zD5Of>hc~%eB+naYg%37iBzzL)$PwkY?ZEe0^xM%67`(OiqLkhj0UL15tZ6k5`n3a8 zmA$<*x{smnd-NBEH6JcOF<;tcO0I#uO%7nVRlJCY#gBfVCNI2BnHeA>JE2Li)9nzi zLGs;D#VzP*h+*N0dLQK)7DrPut%#EoE5-WBmBMnEN4yl--km?HfpEs5D-A6Iwm1jU z>w79OkN(w$B###7w&O60Ta{RTa$&olpK;hq{U%12_ zCUM|LzgE-#P3z#`V5b(Kwz`0BC1PS}X<0OV+;pg`7t2F`t8U71c)1oQU1Z1{RIu_r ztiedHPnPI}+zN~W5$C>DSe6@I;&Xzw>BrGX;-$Rl)l7ItwoaBX#?+VVAV)5(7>Ld9 zcPD*tXFt3mFY2`6KH&UqZ;gmax#VyZU~PPyp?l)j(FxP0TE4}SftCvcTgiveJE_5A z#szag7@3nIv|NYe@Q4GdG3+{JKTkS*NC3h+riJYoQ`IK!P7`y5rbxah}68;?MdL0zbfMaG|~ z)~#lRa!dF8zk}&L?f)0Uw04#}(sF#R+5#J}h)T7Xi5}nl<^bP!cWWy=h0EyY(9mw_ zaujMpJlJKj)<(mko%+9zw;KQYlH*T-5}^b>7Fdx%Q0XR1tHa;to3eug=cmxnzX${l z3kyqPa%fHX?O2WLgMrY2^iVI-X0GXhQ@g$K{;-sz_b5zmBIfrijyKx|2y^PZy2z^|9dI&8;m}c z2?Sa_SZZb0Z&ls;ecSi@HytM@-o-_9kb9JaJ>tdNCYN_kUXM01ZZ8TODdfoQbW=@U zQSoPYx2)ve-d=PIfseF}jZHx(kCkfiL}{+hML9+XaNs38I?kLg&|$_PK_$26S<*p) zFW7Zhl|GT-{QS4JrhC*w0b9gR0DgCPI3aBr(irgijE%(~&zc-}!F+?E)CAhaRQ!pN zk9sbr#?0zrFp&B2JcfQhp2xnrx@KiP2Y$z?9&&DRyhQnn=epQp|`6I$@MmcU}+%Pll}Zj_BZn&z{D#a&U9$N}kR z+t8g?;C$o08x_8mfkc+_4iNzT`IcP*HY|bE%{F75(-H*@4b6@uV0b#!xdC@0-WEI( zkD>HB)jI9|diwN|*Yh^nn1>QD>G@iKcjbE^;3D2fbbut2xqsR0@)s@e>M!=y?JeLH zP$+US0xOmF{Q{gmRXB{u$VJaz)9zGH8JqW#YtLpC2E6}*7Qj}%I9Q50Uh69`@BYO8 zcJRCJ`3^lV?=!%=`|WqwZZrA6)1d0Oe?s{4z19>}&(7X6vSeSf7Y2dSoQD7R2PFje zFDy_3El3tHOyQW|nJP1VwT(wt;!!ha)h&Ag7^Qz|tW+|k0>q~53k@Cx*XD3w15|ug zFaDLc8-kUJ**EBhe#&U!)?qHs#tT6wPsWf3jE7Hv`HF={7R;tyv;AO7UZmd%W$ z3*zSG<$0fu2tR-M@(Un~%|iFoe30377u_Sf?AmI|QbpK!*;A4gEy?@~`R8W+3O z!`-eunx5OGe%DSY&Tv0?`MBlDC*m2ET<-6D?f`RW$EVhML^EF==cPa)J{0B**iWgN zZQicuZGg?qUK)BPny~2u>>SHEA|j93fNN9|s+LGKn?nYnWtnqL1|{(>Hnmn}TNnMV zCCqabzb*fFky_^X3FKt0J9wkZ1h4GQ>jf-^zL)i!{2g+ds=4zbdIH&xv&V z1H05&4UyD2uSnSjetG&5-mcX1^<_1nk{fgIKR#Yc#{OsxW=&OgQ6R}i%fbnF*E*$C z3Wf2`JRXXq3q4^?CjOs$Vjyt=h-!%-r$y_$Uc#P`J$9%ovh7C=Lur@cT2`P#+D$Fo1#S!k?M z-p|sje&53zbnjnb+^i+oOboy>P4upvTl#_A8f*fpuKsw&W+h;E5C56EeaDWw1TScu z6j5Y@@PcIDG#d7-9wgKcso*40qzuG$`HNj$t=qWBGxwoHW~XCFCDZ^(hy$$DM^s_^ z=WRf+4$u~oH=KS)JdcQt?F9s5ETFna(@2UHf~-`N-C$L)o!~E3`M|s(Kgs)R!jd3Bx#Cn79NG|sZ+#6F^d}iV{&MaoD}V~ zu4dicH|bA|7TR3)Z^fif?uBh2_`hx>v^QC4 z{_;hoL?Gq6ao5rmK@KySDU$kfLo)8z3ND6)uK!APfA44co`0Pd_itVL32oqbdS?B< zFavf|Wjb#F9eTi`hL0%i>Hi{5lyoE0af{xcud$IoHYXkBdze97wT_y1(4 zb`znwzwXOJz6JFYQmSLqF-tW!^p7tSIF`TA3y4bBXMKWQD#efws(8dc3^hcTuM2*x zN9Y-OF!C}?)xf2EbVyp3&%YKx!^;N-!IlSPbf{!f{vrU-{Bm^UlEkk2bS3bX3rMqA z?*zyJi+Cn~j3c(!=0S@~u>3p4{qk8VR6;(iHdInfk0;+QiX!zmMAAMgB0Bz}to z=fl0Yv9YlW;6~OtE|BO~nSP0j8%u0_1U>OMiv(YP=l=?P>wiUZ7s{Z`Rj%1jjikKE z*U$QQG`c&CluN?IgQDmF7NTt^5YVD$X2!a|zyGiI^;i7vWij8q_A6`n(ARSdeq(^4 z&*JA%X@G}Lks@zm=Ip5^CvRXwh)YaD62{YcM+6vU@{hJ#>I2}o30}=_Z-4N=`Ww!w zS3&mJEK1%mI?UG%tlF=5ui>AkW8qUgmiQ8Yln?|F0SlKu+)PduLOYYAG;!GYuYOMH z$Np z43~tnOG6s@`En)5Y=lz6o3~8ADt5d;$pUa#5%;~_-Kb+yx152u`@%0ZA-oE}@3UrS zMiZ&U6R9VQc!GMOhcP&bveSXGK0v7Ad2Ra5?sendH))!%KA552%#6x8*N$N=>xqcb zq7_xV!jC5Aj|pJePX#RT>5MquAi1pU@^X$ae9ADun&}=J?nWM z=ri$v`J;16DdbRcFBy0z@))cFGDv{Vjs>Ld^id~)beAr1+hND;s611nzx&px0uc1m z<;uoL2TnF<=Q3&jwJihNEl^z@j8|Wp>uJz&Kixj!(zKylgBQpxNz7N3`U=;%yD}V;a65(7opp1dAk0ye1HGO+TcA%xMf&h! ze__Y0H2!M0s#7NcD@(nYdvj|fBOEXpZLXGqg7^!0d#OD#F(cB`+glb+54dc00VmGt z31fL?W==pQ1&n`IL2pLI=&npr_oE(CGl}yppRFt}-VcB19F|&iC53xzhVa-jB>2Rz zvgPe~*4EahIxT$wP)~W8gE@E2@r)Q#LYk6wM$G4-o5nS{6CJUD%s|=MYD zC-?<9d1?AuVPRDWbmO(%H&`!gxmtg3EVaE$+9VhgtLOk+%7XZnm6bImo20$|&-mfK zsNb9sHNLU2agc8oFlKc#&Je{p%v?aAtEFu27641S#9&A?J!7FetT{~)5EC|m9uAGhnw){FWdD~Z_VUERdYRVC~yXNG!r zOktYJK9L|E%UPDGPYoF|3XZ}?&2cN@u4kV~3x0Cvbt5HY=Ks0>1!~-#p)h+g;ZixN z%xO)wiyM^Y*_$VL(Vr?;FUQ<^(gy0Ij6*4-rc6U1gHDjsEIl0^9r(#!!hBIjz->{v z?9N;*i-e>kKb15@D@o-KLx`+8{}d{FDEePd6wAn1jN>=znjoepqX%@q+!Eq1 zp)W4*pqzx4IaX`;wa*%z(YjeeT1-$xXLg8If9!D0<``n+py@teaYm(Te~SK+0 zg`o;X%J9@z`$=qcPY}yN$$UPp*DozDj?6L4fE^P3IHKjbPVQ=~yL*H0BK*xCF@#bs<8BB9CAvwRv?uMqX`oc6)g>?zqq(4y&u83vGx~jr;&Ti?VWCol^fC< zctw$LJ>uNq9BFB`ccC4sM4F~89&BvefS;1+8)HdL1!TnFevj}1(+G$ho5EKHw}$q| zQVg_&d@MciEWNA!d1I$&-i z1QKbfUSM@TfAMZyJ6=i_-lqPE5i*&}vS9eB0D!K!KyikgT4k2j0pufJVctaM7zpv^ z0~>N1D309zXaHb%;slTl)2_(t;FUnf>{OnASe3G8%eF4kd^izLFh;Ae+J?}2b~Zn_ zK5`WI4S=JZ{pzhnjfj~cV!~m;7O$%^#6VJ}{N~?y+O@p=qVtcuB{3=-NR&Q%@ggZH z(g_d=?@(t$?Qs7oo|%$pkc8t$c#fCB#5lE!aS20{fggM`ChCuhkb)$dg*WD_3T z$|VvcWYc++p&&xJdKETw*l^nlu=>>wbA0tLJ>$Wyt zxhqt3W^X=E1yDO3ce{v485xmyP>VG zAOTY59BnY5=-v>$O+=*Q7%N~?JJb^>MuYO-9_QHuy3DzxeOe}`1mIaF%FUTYFV(Od2Vh#t5up4dMuO9qXxm4wZ^(CB6f z{{s_Piz+Df;;84Q7&OKho9F2HpMopbg8!i8PQ(QVOxNZv`@RTK2*1-|ZV84?mNs&b z9&3g+U;M|=z8Q!2Q@phdwp8y0B13<_fg4_<(L(x`+WovjhbV=eq8}5)hRgxMAT^o{ zLHi=C#N>{5oydy}<@B|cPB3W<(Hn5-R@0Efvc~FjD1-BPWS9~|- za5PwwxSVl=uoYrS4QKxsdk zw8Z^>HJdCkO0aEOPIe=FrcgA6qXz3HIUl{ zKbv6p^5D5KRP;^T0UErf3py%Xf~DVp*l<^Oi~)$jOS~JmvGDXVFJKaB!H$WvSrrg_I*;#f+-Lg?l!Duugtd%&AX!vduKEJ zLJ(QOAu6aEm1$l&4iZ9{I9&jDt$wik=S$?%B=)aatY^G4gr|(e;yoRM(f=6Gu=cDZ zmq-E#H-BVHCiI;_5Unr|7oDLLmB=lv%LCPshPd&e04_gI>$~e$K^O|Zi*6FJ3fpxx zL6M}3nYGK#krAAv{}CiU={Mb7nWkO8IOb28ol!>~YaF zz6HjrVPat51NU+pO33eWfY`k-ThzfXslF)n{q5^Ko-JA*#2h(vB6d=^Ux^;2J=4dK(H* z0{J4AO5Ev~X{>;kJ-evLts=@=RzTM{CA`V|+(^0XSdOXm?~Obj2~~TP;MUk0yxm1m zgO=*$sYlN#zVg588>TnILtNJ*Al)mdZr9>gNCbA*Xio4Sal;th^xWtO2kcIyMkIO< zAU=|JH#mpL1=&tua-j2YqZ-3ml$*STUl9JxKi1nlae=baqfm}vtnOd;J)=&Ao{1TN zP~jXSgFtcCO)$-f#dyz`TLJp)4U}{t>?f^+XIZEsTN~e*YGf~pxPt{T!0_VDGsHv@ z_fvp)TFncS&t4Y@?IUpU3<=g>K_j_chZ+7R_d$@9&xyk|QgYOv#`>SOzVql){`RM{ z!Pm6Nd^tLlQ8LG^>xY`mxZ%9J?LXmK%4gs4oz&$UzWgUOrHvl@bcMy$Bib-Sx;g5) zP53nhsv{R&JL%33qa|l_V-l{+$`fYKocP5yrmcXpD4|GX|jOBN%M63AF7h@>X$Dvdy#F4e16)S?RI#3m z&`(@^kuawBAtkdnYn9c1xt=|fs>YjVcAR`$^yLvN+FlyU=Os*Q57e76)WL?qQ5L2e zspIOd1GA9OAhh_rU_UFmJqaia?VnZM+&R`RS$iz|$p7uM=Aj2x-XN4`wD_~)Xh>`H z!gR}%W6NFTR`9MKobij z>>`N3!Q4DURp%O5zI|?BLZGti)-o1nRXi*^dbt`A*NLFqM4ADO;!YQ#HzSj8ZPl-d|on(_#uSZo%5}L#U zMI6eFDmGPS$mh4_2ww9;Sh1f5gG%GA{wlopH%rU90Fo_7B|mYgvs(*@1OSS(7(NiJ z7OZ*gp_3hob|KjoH27re2&c67p0G{5i8fYd{R@V|$L=fy|h z;3`%#d|6$${9PnD<%jBn`@!pXfn0D*j-3Tl{jFiSEkg~{akXKsSaWkc8QvU%NR#fa zZEz=Zm_X!=<&OhQL8#sW5aFiUeB93vr&qMC8dBF7%fJ}YG@Dt6znv*Su8()htKP5m z#(Y@mXixV#T9xH|`vXN9c}Bb(^bjEE45)#?jE$IL98INN9)NsagC{Z!BtxDB?fYbm z1!GlN?p6ja`9F7!Ea-|o(8S@6C171W_|)|Qdp&}WD>%=--umM^)UFPQ;R=iS{26cUVT;Jw1DVy!-V7P_wVDuO%C(fLSiB?n%I583zo9 zsZ0&K+Vy}Qz{%A~l(cQpFV$jlSZ)(t+JdHY;4A}Dm?oed*V)ey0OK$t7Y+0bHIT=~ zw-p?2*efakEcf@NtHH8URQk8B#<(>tldN9>r6-=_>m-Wl9LAN-pX?k%yy0D3`7i6E zH<81#>jfUQbuA!q(5$HC&Us2OYQl4@ugOBAZuIJ1k7Hr0j3KJP1duB*KcR z?=m0=UUoe?I$C1VDg>NoihxEDASi!hhY|t#YgN0lOAz1^zn=MSqDyxMc*9>_xe*-NSLkD@(+23_cotVAT&0knbin72pSa*U=*@y=Aw*dOP1ff zgd~dFKGg>%qt08zy$(u(0K%DmAZfY$-J~@kd&r*OH>(NYI>bXvRcXS`G=RaAo2`Pc z3BcRq$4NCe$SMdXtrJc_{R2oUPaQ%4c}P+4_7-qQ5|%a6F93&?|n{l|duW{7Bh+V5qCdQ>F{Dn)7jm|Z=6U%H5z3gC)8Tt=2bb_! z3PH!_dl>g{}{_w0qmX8>2ya!S1 z1yO$lM7G3iYzz!@VEzm%NM5T*Xlf>68^YWa4dxg2AqzdbM2r_;12#$6)s^>D4miC8 zG2vDG^5x6qI^%v1#i|LOpadXi08=^LK_DW5>4Tz-V&?=v@jo#I21(B60e`S`{uP2l zx4~gf)!!E&&diRh(?KQY6VH2g(2!fG>b8e`Hp|~t2!A}3r68Tm*R)tS&(~LishC1c z8zY02UYYk@%E;64)>8iQV%B`4A|!A5GL1#Q4b^z2@u+$NzM|p?e~D<#5)x4!cGAxa zP3aze4VX&Ava+&G0M$#mS48_#aUbKoxB`BSW|6SUz|5?8ejMK+f3xX&Q}mH-K!W@b ztYNx)c<6lO$*^J1&me1{w^^+E<{FrED`)-$m_zz==b(&bLg~9)v<~$wF-g}YfRx(b zdPSB7qrND?z%zc@9IO_;afqb!@5}+=^Q;C98pVGMcYCs;a zPe0PEoXmdyn*vs+HiCy#n+N_v(@4V>{tA*P$W|=iNetH*8Xted#yH@g%J(Kzz;A+Q%$5k&e%wpB<9fI(Iaz5$y5fKOIN+Th4c5-2mi{o2-O-i1 zVc3~f{NvsOMg1DP9Pw^s=x3i~z^Q#=*#qCI>D%4Z7o?bXmqkK)36?v4^WAolZbq|` z=2NX*U+M==ckss(%D0_8z(*4&=v>cQ8VJ3wp7bB$C)vIq4~c)gjQFe~MYOynj78935Rq*h4B=Os3x5B5)XIAk=4#1#q&!Z*^w8&q-+9as;4%e%oVbY!+st-&vpxGC~_4fj{6OwO|o zl9vbV_iUl>O2dT|mOpi^z6`cGE}j=~J;u56DbVv<&+5ae65_GQ&S-HXH9rdKY73On zdG^DT>KO%!$IbH}V|mAE5z+g5L0vm<3=`k{HO1Jz_v;-|WFljq!f+wP@b|A^5CPo$ ze%43z=mrB)g9Q|}s4NU2z6&U9TpeO(zV68<6d&#Cu{+I`%%+YNp*Gj2{D}_d8)zVO zav7Xc1c)U6fHz+OSL5a}25E6dbb@3Jfv%*3|1CNZ&=B)<4EBLQ%J{XQ;Yg&^%O94W zr#?4DE<>@`YlBD^L8M~&$&riT8ZPg^ER${5)*03(WUyS2J^J9q&28z8+4GI2eme7| z53B1x*L|jZQB4AUb|}nM03s5d4~$QJ6Zhiojiaqf{|#}}sxoaq0*Yo?7c-M>5p#EM zIBj4AfSF1SLF9Mmz>KZRWd~q+YYD1p{Q}{3QCZP^0-NVN8ZUxII_ zAS0Rn1muLafCVYq;>5rZi^Dc=fYT~ep#pKGar?w>N-|ATL56{7A05I!r9Wy(bP_@-A${hAa8D-b6AyNE0V=)owV)A;%ucP&szl* zz2hV`g39E%eQBd-!~_mEugejW94qSfTB$rpk)+LzLq2z|Y=gE>K%^vpuQ^E8UcIQs zcn)<;jo;Aa1t7vpH<;*6SUp`=$nj<3wC8E|u|<6|MKgbTyH$s3b(qEPIS>(?fgM~r zk`T&R+%``)%ELXXvE_Nn0LaCz6R zMb_52t}MwnS~5z{4~5q}Z+Zn?%zWe_n9FSF$zc&Uxj}gsN7q?wS4y?aYfkemlsYo} zwi--U`JHa!bP%NIl* zw09~hzg>27g~p1}cpZrUY-)Fnr%{Rt_sIe>CKf;>63+Yq^N>}$2(L&rE&s8Rm%MUx zcv)L!^l)F_LhH_u{~5!)hq9~Ze7=Eo0&N>fxN5#Q6tPSAvw9nvlkQ|vN=K2@07qqy zno*(RC?!4G7ISfkovtcB#OpG!OO42W{i9Rp!|9acz$rxxN;VSB<^N*qE90VUzjc`z zLZm}NQd*=$Qc^{_MYF#b6ln@E&5Ky`W1nIK||9kJV&+mME zKfKQH%>AtUTI-7S?US1AE4YCWXu|T*oI6?R3m|&Yy%EJ5_4K7>C=0IAkkW46s#&sZ6a)D1!@z?uhok?FZP9HZHv zKU4WqGDxTb*j88$?_fTEvBL}cyEG`ZR{kcIea5}LyiOn7=JngG zYMW8SsdW+2NKTOn%d_{}wtjavux9ou7CZ=Sj!{?K=3TGFVJQiY?En314n#$6=3)Eae@i5>%$Hh{c5_IWkH~!2rE=eVpVily@S&92h6)CCmw$xIWT>+Uc?lfctmDVA?`{tB zuEFs>6&?n&JzXtVqTBWF+#0s&Uj2wfdqsmCEf|d}NKHNv#!-V70u&#r&-eo73(blG={d*Z#-ki@$3{@{qmYg@Aa3CrJk6m{4ZlN-$52fN>08t!t*?K z_g(%|8QKHY%T2zer};D2GOP%amu5SnK?WGJw#r!}!QeoC`~H;_)+4{6WCW2}_wSza z;|GMG2Gr{@<2pH~GT-Bo+Y_^U`0(!G<&72D5nAT!l(!3iD17mifQcuL??RO>3qIdY3%S7+Q~FU_V%>0>2&AgvVGO-s41y{W#I_0y+M-<=$0XLC(dKV|J&>Ha#^?wQ4g zHxzwR2$Ig6Bs?4(s~WE(RgJ1-Hi8*Q$*E3uU;TX;gWKaN?g7k2bENDqQ8d-Y!ru~V za;k8Aqw*Hb5RclA}8O*0$O)aDB~idF0AwPu+$v|ktLg}L^XbGCIN&)n6z zG~Jx|?ME824C}c|u+}B9Hkl_q6OJEt1|GBBT6$}T-Z|W8N;9gmcV8)}+64ME&9WoP zMpoz6<)#95*{Cl7a5WpAxln@r|4&n=V&^0M>e|{=Af60NpUC@03%-im^O>k6iY&?N zm-f}(cRaY3;l1&tVEm$VwDj%6Up5guYu5~Z`3Qnc}uTF0y4cP3F+T{i0{9{u>KM)V>rF-l^`7vpxV_?R69!ViP0j8YTL+)alTIZG8Egg zJv)W2d1u#QZ8Ofwm&0~KLu30LaSxV=*ZEePio6W^brjAOo2A|Eooh=;ngKCJhaGQ) zfzK?zJf9RbkmLnU=D#3sU_7k-`?pp7f$mKv=0s`li$H~5GY}Uej>2&Gz;efSXYM8o z*iHTTUdCCrU~J&EqBT$@5k zG!7?8%pnfGVRKgov6sC&jw4f=JDDS}CW|K$^4P#eZ`Kq7)ZdJmlBn}ggl|d2GVH%a-2s8o^oGgsfb*Z9`@J~_(Fxs}xXVrX zL|k>VR?K|L1kbGU)8{vwROwa7ev|4SHc~M!4X)xjA0QmIMXWj=C-prUR~(EVV1&OR z^4;ifl!HUO2=8by8%2D`@3A$Rj#;}nGO7Z0!Gn}rRwLswHBCq<`8mGk={IQG`@T^H zzsd>hJJ9m;YyBMNj?uHz?YvDxUn_=_3B&WR$}Uee$$&7br}cgYe8O z+h5`|ACOcmN#c}Gw_p1enkRnmXRJ!_OBK!UX%f2MbSkfOru)AsXSP;nn*KpMJ;80h z>(q?)$PW|X6x~I3nGFk)o_|OJh)8ygdX%Ss43$3Losj2iRebpt>b_FHr1)h+BPt2F z4a%JFC5mLeiqL5Zh(+LZF4&NP1mV+9)~W&Laqd9PBFaNO8k5HR&Ak=P{?vYSb%Qk5 z-~Pa-sTLXzxTk!}X1g*BwW}U=WoCAkONtX^`kAgS=&P7){8ft=B~utFe(A3~O*xlEL_N?H zzU%5<#K2GJ6xa|D%WY`GEvF+OkjLrZYPFs}HPo@EVY6Q8ora^}7?AoW3%Df0a31V4 z{?nFxHp`{=^|^s0H$Wbv!)&+39fV1N!q@wLd@)Aua&kS#y{GZ6cx3f zsV#=F5}h?S-m)9bmd`h@rH$45zwy(n#Nt(07&HI6vs0Y~4>}*vD6iQ0K5~}qx*;Mn zVI?iL%h8Ry?SmbR8w1g zOP?ioInImxe(m}Y6B*Q%ss4kG-tT=&XbUHFhgx(1Yms(ygs^PT6B>F0t~1qbET4$B zO1s0t^WFL(0@=eSs=Bu#BA6=_s0Yh~@>u1|c^*bNYMyUfv9@6Mh#N}kx_#_VjehH3 zpSgOzF3axI=?~S_4NjDIBX;NrD#qu<_@Mt;;$W#I5KuoqjE#tuc3`we!?xKmOoO}gf)!@YvtiH38V zAMd9e$~CtI81;XgI~WD6U}CkqR$ZPY5oi<^Pc*!x8 zJshiqjC9H9-wEYY`^fcgGzY4*(Qg6?9Wbo}(SNrNB$Bn?y5G=7-e@kJF-+$3W{I(B zK#T4oE{r?T8NM_oi7>3N34~O%RG7x=l5r`a*%-7inHeG-|wp1V|afP;otvwd6(kE6>PBY#{OHrVU>zYIPWKYFqM|CHhFDuCu zsF|g!N18S!4#fzYY1v=7GW+P&Me2s$-d18e%*Eh=mx&l}`kpDJf((d7c~D%7dDolo zJ^WOZ%~0+2VVNbc|e1;cIX|EyBisUH~HMjf{76#KIfnRK#Y0Y^@j#?kU1C< zP;GGR^1C?<;d;ICGg~vbVnYzh$PCd*^g!5rf+@z}c0>fJko^P=BnbI8G)Qr%zD0u) z9u6ty(JBqQp+bVNYTcXrF3=QXamBtZ9Bl>D(65b2xNkei#DF)PyzciP~XA~M~uD?{UBl%hlOKl`Z>S*Nxp}=;Z~2i3 z-NsPbxv!JHchp3A7Xlt!P)+9XPl}4;7kJF7nd-NDwCp-P^-XrsWJNX3caz`T=Gyeo z4Z9@lAi)&-1P!~i`A|n`*{!WDOoU^6EaeaAO>Z9VcAy@LU%$@6a+8k959%%>>t8CU z{G`8SJ%225rp#|YsOaE$-1z*hdjZ8E9s~EgS}{u=!NQ*DH?nQ&W83IE7~qT-SMZPy z?-1qm3}HKkcb}q2jV=t!bR0h>`yn)_!hdeqjFn&J&`1h}KQ*sWUQ0_0grf@mj?ZFwFcV(hFjpF6~iiB58GbkrVQ#-x(g zgQ)^DGc}}QVtNh2#xyGxMlU#Cv{5IF{rP;G+FK@Tu`hEPy4(5b_*rgb5#eq7cn&cd zu;VGDjLW6WDU59g%bGtBo&G7WzZw>FJHXGXhyzcd!VFt}^`3*AxLyFpgaU980R!g1 ztSnCDIKISk)%YQ`cp>}>+XRIn1`I^OOy-!dCd3P+ibk0hHw#9Ils&d|Nrw`L;s<2( z>cqG&%#6IyRS9nxVnRVB65g=x?wjAktJT0>CVbu|)($j7=`<;$>IyKMNurw0Jhq`AXH3o?N^7A^p**C-de`~>OCr- zfx$~h3vt@mT_KmJT%rR^xx$d3wH;-D(j@AaC}XWQ^tD0WwJo99AJNgG^~S!L?vR!~ z@`k+3FJgx4IvG3P+JUhfu!Tdz6$taTIU}G%#_nx zT<>IF!wXuI;YiKv5s(+R>$M`r+u;{_$uhif4Leg3MW37|<@Osp>5#(Rr&2_KPs<)# zv6ge9&-wp<@GCmuWf|=O6C$>WSh6}>%FP)y=1U~x9_U9lXNoTO+II|P zkkD1@S{(hJsdnde=x3!UAHZpHY<4}T*p(NxfO=7gLsg&sC!wx}`V7TlZ}LPQ&b@p+XfLvQaC?Q7U0G-VS8sQwM7f6k ztk`0*(FWHsskVA%-ht|gsOJVVC7LEYE%x3=G!?VD`9lvLmkkh=J zAne;2V77SZExA<`+i1i@?-7T1QHVDd4kc%vnwi(6R_D*HehB!)2eR28n&Eq(3T!>;RnO*3@y7W zA4mI+a0}V5X=7-COVcrWskFcglyf+IrqL7;!g{+WS-fn*EMlC%Il3gj+SmV@g4VCE z#{udNmzlJO>T|2E*be}@{z80DAH)QP(p*IXaTz3xqo}Rodaz@ChdL1Keg>Po=B^sd zH|fkTB>D_|wz%rl5?S)L_Ll9wR!OnMvjIMPcnIv-7<#}^MViI+IG^Y_eXso{~)r5;%1k*e%G?DLc6$Q`b?+BBViz%*YVc7jxYCdBc5=OQYF>r8? z`fuWm(Ohlu8FN}rDUYQc)6BPeJjFJ^{t!jtExD z-|xr|4r-2y^JBl|*RC*D4*>LrS(i-LhJH^@ny51C7 z#5VmDT%_%bsW;0XxE$QLLf1fCe$>k%veVGZr8?zpqBi{d&&|5rKs6J#QX#>UUW_uY zm5nV<1nXKIV#Y7$3x911NwtUUpiru*%DFwfTCp|M{qf^UA75NCLY%U~WXMn^<60N`e@osp7ucDP_#B*dw+?ky zv7M8%8<-;5aTKEp*5MMz$ja*L>wAmL91m@B?%MRF;yl|~ytg_C-PSwmCEH7C{)HRL zjnykx@r$cMvGIIJSOqqw)r2SOZz6B^sZ@HpR^pLj1H~J7zFX zX5n=Mo_$!OqsQufvHBFM!mRvl0)geIqa`UysgwD2d3{5}Z}1<@w{m0tT!owdu9w`Q z;i=cTmI5?R3x4d+;cjX&x+0ZjPr03e|H1X-0j!a4C)MMy4xdwYuRPbgT(H{_k~4RC zZ+65kAAC!{fbjhU4mt-=s0s4}FdF0w3=Etbz&uq|6>Zulf3d}`nt7U*Qwf+$PtQKM z>N)>&Qj^d_lmzEdxI+cvQ*juWAoe21`De$PdD+K&fz^$yB-Y#C&t@ORiT#K z0DjwATT{STn7a{4cu@juGk^E34Ll(PdN8Ggf$J>-{bRMkT; zqraf>`t)?l0SjP4gD`>>zqumLhZ9i`TyS{!t4Eyn5xd4ipR+tZMDi=UFIcgNk@(AUZ-Ee;`_CyGNY zi->*#{(6k}>E2YMx6<|wstdxk&GgG3kpXmne0J{V8YMW<`^SFOT+QDQ?mirvweN1P z<1e{6f8(*6o2a$TJqu)x2sd>m0mWzd)Iy&8eUI7dx@jm=ipOo66gz z&|?}%uDY!o^B9z3|Ni|O zEYtD;Ag_X%8ZvRYa^&H2+F5X>kQmy7@llV&3D&?eo(j7Kqe*R6n3fpCIb2{Pn-jzL zG4x5nxjDRxAYnr6H~0yv(a=P?;DF{R1+ZZ*9<<$e@ky{B$lzhyS{!|tQ;_}DicLht zb)Ao~$-Xu_lfa{TI)ftvQwrN;b95-0rARk_!@V{%jEDIKHn)S2PM#=|ax{|23}xtz ztx`Fi!UUe%!5xVnU35ZO06cP=-EQ1jdW z`!V1D6~idw{+Ntyze+?i`eNg~0?Csef zLgn~JR|}(H{P`l1_1*U*__5LIc#n@L#ja}ncfr9`5q0@;_qlvq&JxLW9q5(kL@uzg z2xb}=sqpI+(gQY+8OQXjowbQFs6$TM@6vOA)N#ndB&sI%jX=4XsRn478?=8sx>FNz z_NfU7ZDV74Dqh3qNSyPt^O!nUx;|~zT={g&x$$Rn%7rnkSY7j5jf~@RWBuu2o%hx#eY{WXW^F+`0QMNL@v2(xs{8{-|+BaRV z3Ea%srQ+qLlKWLE4abx$(gjZu2oyCpPUR$sSWW{|k~^fV_QGR4dsVqS{rdhd3HC@} z62iG;%<-X{`LuS5`R%;X+9vzP;A-r|c*e;OQ8^*EU{HbRhp|D}aV+3yaSD1~cLbmX z`#HoepTjGHfBl!vI*C5 zU~@+EA%2kQvFj+$`<{cBDZj0UjxV>>ix@L4ZTK_r3hp=;qJGeNl>KqzH7pl!)q*}n zuuuvL7B_u=e1iD$eQi92R}AnLri?0vFzqe2hbfGBtwVvm0|oYiv#3iX7<5@>0W%F! zl+7$U*QO6IW`Wx^%!CqJ#ZO=-{V0k1&-z#J&8{+js&Hh!?baE4r{_jsDGR+XqKavM% zxRW0LS%1W{bj`T?4k6xeS!9(d_uEseR?n1)%eRE3vY%e2+%7%xvj@uyH>PsvUT!ck zF^QgcTNxsH(zwrISo!!RG!)a2tbKJi7tRi&#`S4qW0J>5LX)Xt5)b97DYX&W*X@5$ znR+_kP+M?uZKf;ed{+zUIhAkH??GxajnE@aXj%Co+1h3lv~T9Tn7L&9Io%?ADT8nr z1@|9^G{67Rd~_na2vX8D!eh2E#Cjms)aPQI6?MGn7~kS}#-Vy1uZdCrfM6~?R5~O8 zOw?FR2XRL7z!x2&WbT-(`JScm4+AUe2o#@Cm@89EcT(9Ug>Y&%ns8Lc5kC@`xKAO} z+QvB@7Y4k$8zX1^SbgZ(3*w2;1~vNCdmpiRj4F91OF{aPYqbUBb1MBCQWE-i_^SB? ztEVW%2%p=ovJD@aB^3M++kWR6$j#o1)M!F!=c8Ta87Q?lsdkhxR%-$qg`_d>wO(-C zFG?$7R&|V(TL*yeHWftMur8xM-gmC^!bGDt9SD6tf!O9>C1Q`xcUqB0xv8`DY}+A$ zIpQ+B;$eDPGj^5%!oud!=y{`HzTN-3r^5fzm}z$o+=jNN8!HF0y2PIT#D?gu7m(FF z1I5n9r^03sc1$o$gQGn#fv#+UluV2GTn*9V19GtnE)UbjpTNtbmmB=bKWYAdM2;nX zkv}GmMQ4~Ob28Zrs`8@$732g&6tL7_PA_~0=zFeugv)$tqq*?6?Bn|~dJ1|eOA>~g zzuxct_T_KU%;7+LGEe`q_}!wV?@WI(;%cB4cR7OEd*uzF4|q8wWXVBWBKB;D3zKmK zIpVlD=PBEB?!wi=o3$q0ba7qYytm{1NHj=Rikmy7?fiA7-(LxRB> z25;AulSU4CpHua~t**S4u&|8dd_wDPN3EHO^LR9dL67tzFYO`vtcH7!5qiUzGf!<5 z!p5m#!wm{j=Wm&to7c`kGj~TA_QJ>+sE>O)a}>|nGH>16nlAifhHL11ezxPuHk}Z& z*!gRFrh4Na&V*e$z4KN0WRV@HbdU}GN_6|t;ot!g5=6M*DwU(HZN$?*y+rE3?P0n! zFZ*S~k7clChFtCc(qM&G$zL#?V%hm=UUOg=2Qo#%kr1*{ytC;OZ*Qt9WFaGEE? z4C0(OeoX%R$H?jMAD=S59bgt0b99Cq9$D`~nmrWVqIL0~Vrpn(@0j?FqjwzOL4vel z<}f-@Vt|P^;i>c$7>V9(ka3z?43Xif_a zZhvnRk=~>QsYZ+mZBSy*>A5}XURqC7`$1!6WNlqXaCZi`GkuME$b0Y#)*NjI6ot9`Me2l)X>*fL9Jk<55)Xomn6n zds0vr0xTGJ4+>5MeYDW(R1*{IP~i?Sf$(%4UpCN&ISZ>n%OilfKH81jo}=n~QM;;G zn30aU2YUV#csv}jl6ls=p`^xuYgM-fcJjKrSbR+%T5Jfjuq0USU@5^Fgz3lqO<|1x z>@kt0f*?h-~`&q&_>-d+WM|{%gOpH%2x61j0_LlC^OX9;_utkSwF z;|<527?s__jqHpyd-017e`aGv{KXHSOshM@#50aovKS|dsZ)|gUd2#>sabATc|-5l zZ@ePRoEt|nG@{Py&Qgg)^=9iIm8_M<^XUojKO5xzW~U-&X^D9NT!)ONI3ixS#`@8c z{JsXf0=XkMO1B(VF7{2;6@vaEKmNq*P}#v*;NTGhi27e+jDS`q&NRg<)aFxc%+m4+ z3#-KCf3Y4(A|0G(bcIWsiys{|tmMWQno2TV*S+%5PE{@jg%Rm$4fVSw8)H8{AwSm; zE!i}Lm9j?D96-G$Zq>Pa<=pFk^a!D+=g>E}2e2`8X^i-Zf}aZ$vWl3a=ZdL@%D73Y zbk%+0S_aCO28U0wm*lOs(ryPLCw@Q{)kS7lH^kB9%Zh4yAQa34aO+*MC#X`JOKLi- z>m~_og~9HE+Fiwq{?1`v^Zc!t-TRh4hAVkfS^(v~V=3Xj=w5zl$f+g-y+nOm8x~#` z0!Fzo{qopVmyH%h&u^d#_f<`9COG`@p`l``$;cenz5M)HM*Ue;$lD!2^h44~*d0jH z5oC%JL9EE&A}bI|Z(cF@5l@6SliDLxMGWbDpC&+_Ei~f0B|JNMQmSQ!V}U z%9|JaU(9*{01ux+T^M_+GFSTjq($FJTjfPxn(kE{zK@0=S_mPC01y4UoZgAb?&rp^wkD9vfHwh_%XX|BkVCy& zj;3TP%HRgzPob=crT!HUe%6m0?BB{!%Y@h%2`hi=qy-CJA?|(19kQu43y9*Hi;g?= z4{j$%r9)a(7C{6XO|8%I^#|SYfv`+!rdxDM2s29J%Cl#{9SXxedd=2zrH^!R2c=oh zSCxgYB*^N$7MT})rcd0%18X6!;;k2i9S)-y`8EjB(0)i#8_I0&Df_wGFC%DTW07uAC^QvjYcAo zj-C|&GF3h&)m1Z&Y~pWj+H4&uo1lq*p^vQKE>N}@>=+kwQ=oHOuAs7-c@u;$2w+oC z_B^b>15E$>yN1jMFuv4O7y_(zJ!L6AVV&-bo%O)sMkF@kbyi%p>S#xg{KXEPN*`pr z;NbST_hF|4Brh5~Lt8$1l6;R!V#%3%sR8JPv$=qlGC1aTna}~>(a*&&z_a(jBBkXD zI)I`oj@A&e-b9hC13N|qU93d}L08yVI_>0TwtarLQ7<9cJl5CN!W`#1KY$y|!Fd)x zIhadQgJ=-Jyoc%pv7MRR^|+na!C`N%(*@@&5sl5P{wlPUO>Xf!e(*?JHD_0%2yL9rynsI^-WV)3>&@eSp_xq`^~EIfly*JRBdO z1FeR053Wfy6G@iTomu&T{lVrJ8h$Bs9ou>TP^HS#Jc9_ldwqXB_k@K_@}6^E*ST_k z8OBtjZxOQLcv!gAA4iRZ&`ldms^c&vutNtnYw>@S78 z-)V!sc>UJ&SokcE7D$FByw1v5+%j3;$918>Z{3BDZv=JHKJbTknZWwK9yl)t7Z%|} z!cbpNZFTGUMw-gXU`wn!mEDj*?&?xL{G zb4Ub}+CAnsseKMPTdCm69pz&si|o4%=rkNRQ!<*KzUCPs(HGZryluMu<|L$A)Eqv^ zUCNyE>di-%5t5S`c%fs4LTRs^?dOa6Pe#~B%woj6pVp*qFawIg<~>#B;4;iu?^b+O;0qc@V=UayRWe@Z|39U^_bor(~{89%h|o>4Na1 z-sb5QVn)eea8ylaCHs5?WoFSl5u+px_;kL&n7RbS3_3-x!@_RE)b4J#Rwsv>oE(Rf z_iMOS*K=k)G7@aNmtN^CgqJwPZ3Nz5>NB#b_5Ls78Qu}~0cP)mrL0G=+yy#S+#tK* zLWWxBC4YFj)14Q1-&ng=S663GFDk|I7^K>Sz93;r66zToyzuD-(H(>3qoX5Oq*T^~ zS-&;^?ApU`mWNMR1PhpDFcyiaZ)bx4lnvGCjJqJ$YyHPPvtRT|t^h1KM2BH`C@~h- z3)!$_4+k8`pWqzbJx!aSQQZZttPLEq$;7C}$5AdC8x_N*cPg}Ddi5fWH6fQY@@1<$ za=ZzTLm2BD&+xo>EuuiWuisT~rK5aO_vdT@bG`&aN~YDFgF!J%D?eP~#k zl8cSk70Y`Iq@8!07_Z)MJqOETaKzP+jMm|Cu|61?$Y2vhFE&FcU4c#9c}m4~`Z@@S z7JBkyV%k%svJpl?S@CYLN-aI}*_MFSm?I>8o0UpV_=?n?I$s}2G{X<-Hv7mYXY0sL zp1UjOm2egP5Z~?veh+UYvuKmAOrcECiHl?LC!>PSw>xg13^$LgM=_DU?;+Cn; z2J)Gw7ezLDBn~htO))im3?#^tBM?_3=Tmg5FJ~j|=cmq4e$%-%T^Tjz1-=wWPQ}Ax zh0NAW7QBa#9%Ub><`b2{_o=KprAPIAvgi?YsPPuCHqF$5dp|@N3&O{rFHjmq z8BH~@`g|DiHW<&6%B7Y~daj6#Yty6@<1OiHh>07%@Q`CT$&tvONB)#D3i}3F@aC%_ zge}kf#sMSj1yJjZkAKf%-SEfHP#KgDN?m0u;8oDXXXqbd{c(1CntP3$#q{jouu&nP#8?^}VSRxRw~E|Gzo$p)CrsQ8_eQVmb7= z+BnQnSM6afDYtB7^PF|U2*^K_3T`F;Cp1c>>u3iS(B+8)lVdB%TTG+1COO9HFp z@qr7tZ)G(0FTV`NGS(;I@xc!cLB1MBy%NPuW$+_#@*8CJ`>x{m^N-(gwBO%W>B#?S zkF1p}+7H@V+07wF*`_unh~R|?uLb6U(z3@<6cktifm5;Xsm~wd_-bMM34I&^NIiQi za1V(Y7Gku)o($U8*g6=sGs)xrpNOb0n83R2M#!1SEqR;%j)|=Aj)#L1LHPSgzZ+eC z+G-U-6H!kKCl$>*%%>0d);{Q^@dL=4Hh51r^_fS~nZk#pLe*uX{4$B?!(AbmdA;+y z57jsfEbbgqaiOl--^<`X)0uDMhIwDqCSZGqCX~YHX1Fu{F>r!Elg=rVUwap z1-L5bnso~1vo{EN#gu3NPV>-H5i2s#T*y6NUyoQk77bvfql74wmqrj!#=c$m!3<@F z!7jm$*JPuvhQFfLXq2dRoKwcA0R~w-+}*RUM#Ao1gzeP_UuoWqY4#^jM20B_MCova;GsUj$R6tLZY4P57o8FnIx6GLjWSXl=mO529gq=2sCP)V3^}}t9$i9Wk zpDw|b%_?|cU?@0@gN@Bb^tsk{nC~n>xF#JdG&mTIVZxnU{=w@3Cg!17an@yUX97nbo^ry$hPE$hM~rLnl&A&@@~bOkIrPHJ{q|6uVG>i~YX; z!F4NNbDPq~zU%d={%pT}pcO2x0!;}tj#a31CZE4I25J7RPZ~*`PMw|v!E8Eb32g8@ zru8)7O*!ic98Had9307tpG56SvIuyj3@425 z8tUliFehbGx~MHu#mXXROHzb5%)Zmd>S6=##Pw|9G9#gLkB)FP)A=^1 zr5*n2P5*5|Crb5N2;%uy-1y%#wO;eTZw(@Y-N>C=W0iIZ7t>u}XbbwtON`Q9!7$>X z#>8?8t3ev+(h!;XhTQaovMfZ*HaWPYERD-nn4;6Jkuu>ypDeaR0ue~&8|SO_a6f(n z!v)qP=Oa{mRQJBi_U+csD^WHudxMBGHd@k3kW&C1bqmHcnC$K6fq^+zK)LF*L=kRr zqdt?9y$b`k!_Q~t8yn5t4LNa2Of2}Jot5e)bhFdvod4oJ`i*dSAilXsWgf@GrAN&* z`Bqp#Wck0z>bv~@@$B_ytbw2vdIzS!%tAtxU?o!ca5Hd#H3LXfOeh?N!_wLX903-& zm(<9L7Rd`rgC_8j&#Kwgr+s<4ja(*CTMvK-J~Gw%0J1WW&evthhmMQ9F$6Vm<_yes z6V+spwtf@f1X=b*{`;G-1M==#lyF#pkn5`c9qNGKQf^67kZU)6R*33HW;#`eyQ`2RjFq-SV(b6(t9<4o#a8CJqE6fpStOf$Nv%0hU)db z?bPh`+YYt>k(tx*6k!(Js>0uZiPKAcGSz5fdh;d{vk5Rh6#x^7N*tds=5+U~1P>!w zl zbF&EGNb(8KSJyeLz_-n&r^Qo3reP3iJWGyv)Wvj6cL{O?ZY9jWcdibP9!$86r27IM zjd|3)<}|$yV$)jB>5@aarEdh8iP%&qfc|{bhTsGSI|ilTHGB>Wq&pud$i@6xHyh}8 zR;gY8yEwaV+GuE}wMLDgc3VH!NhUbom*F|)?#pTzdrMkx*9vX}OZWp}f$J-b@B(0w zK(!ZUXEew~ZH5c&R_{_o@<#eS00QKI+Qc~SnL$Zy^R*4EH+9wlHRN*>bTUtO&_vRr+$`zx*xMQo?e^xb|2qP?+^%cH^ zPU#Ac0tTgnp@d}t$g)x2(VtGSv@U;RfT+pb^e#;o`a<;ZMcn0zK%yi0M+# zo$vqroK%q4V2w>SYFPgcn$1EIJYHi1w=b23w-V^NVL2UOMbn|aEI<_DSSdM?-@-%G z6Q(|!u=m?dh8AYvPKl{@p6=#t9=0+utO_AjlXK+^qUujI8aIwcDWm zRAa&CTT#Cw-<~7{YXfuqEgx%F9UG~hsc(*_vvgj_gAOLJVe$`VOcg0lCi947jf-ML z$8;W0Xt1!@-iIb2J^{nQ06x$~wE?Mfw|fI}TQIpkfN5ww-^f|!q~1iQm0?C{5k~NX zO`@;!Bg3Y!ygJ{NK9^fk1=>Ta1Y=1LQeJk0!ke!UPDI*Y>tg$dnMIj%gh@=`g4h;6 zDMOT`j0Pg5&sO&Mo~$X8y_lecDDu-?nm z=|L1Z^oJ?V*2I|#mQF}@1T5mpio?P;JSA$&LiX#5EbL!=|O|42;YPRV_v{86)`+1P~k5xTkk`JeUH?su`&~@1zm|3n+Q=>pv zh9kfCjjgAu3@>-Lmz*E;+MVR&Rvj3c+kNWHl)aGB3QcGyoD$H)PtVLm&N#wab8KAP zif!Y}<^D_Azc$7BurJ$yv5x90W81SZ2P*!WTsz2Uq=gMzH(x_&lsdxeCi*g# z&axZ+X-tX{jESn2gLYsPnxf6l+tmYni^J)Ow=gj|xSt3(8Lsj@8NzjKsJ!;46m-v- z8(4GWg1?alN^5C>oK=)0Eq&&+H}%|CGd(`G{70cW#%pu-ua(2hlrRoV zP8UH9@dZ|Ng+U?jEF4jsZbh0;G0CR5elad_!_M97Bj=~gX<0kOMzm;c5)j_LX(_hl zF&9K(WMPnByeRgMYPrMAe=ziG1lZb2HoZe;m$k7AC_G}F%gt`9BasPt9I53WQIH3T z!aD`w6kCs5N7f(SC~5hm{59{W20cb|phax*>8U*#U_s6%*9rHt;k`Cn-maF(Kocq6 z?k3MC%mR|~YE8q1Ec6WdoZH-FWKsr&r;c$~^wf!HROC3w-EZgdUY8~b{vJa6edU<4g^uwJ$EZIi=sFImW`d%C~O-~v^c^B62o#chhybBss zn&%UdM&K1mwHBZc|2oAr#p_i(l6KC~8W^|@su5e_2w44H{HTwG5_XV_!s~-^COt$t z_=FMqzN=SzH86ndS};W6FdgXiAhGsB=NyxSAm4fm3@(0QJ|1u~x`s$d?}f>ARrq{=EajQQ z{cAwNB;4DUT;ps^A$o)FS$MdXd{pj!PINchvva*ix=|A)KZ2PXH7`c~QmRqk5nE!)XRPFMLB~nUdU-L&32ycyZw4StMUZm%fo$z+N-{s= zq?{EM31enY%XTN5R|+~ysNj?31scdBh20(SM?~?(q?v*>`sCZYs2?41VAc=(wC^N0 zyde6}PW8QoXh83L978ozuANiOI^X&d5>V~n5#MzY#NMaW$l=i&$6;SXG_a@yiRbZUeuF8J>`vJ9*Oj^vVK_~z^OT=QE(!nK( zO%$)Z3d6;iNaKZy3X7}2_Q*`d${oBuD{EWAzI&0Z>mG&BKe12{d z%DkSzy1M_TVugRq_2dw|xZn0_2G+*F&_Cg#z|ch)*WiJL#( z^2s*ozw*5$QnS0q{LAt&edUHSGwxA^%%CK-XG%l@@ujOvhw2`+;tS3&KWH~ZrBiUK zPX~}lD?XF99pVua6Ptz^HFtv>tVF67!=P0!Uv zLgRvyJ;?NEi%PRN^|H#N!O?H&Or1{>_*jIl^IaCmHD+Uih&D`mRy7lO*jSqsDg6#< zp2$S&=k2{nnb+4o%b0Zyv~i-A<96i{ZV$0vv`JR}PJXV8Ghs5Y0;f_00NQzCV%>_{ z&uoVaF$;a7K1)X}ffR}su*}dEAuO@KoDPdj%-mcajR~TdsMnc1wCANevAwx#wa!cy zdWmgwp7)ftlM0N_CA$>7f-N)x7CkPi;~qFok898UE9wsHU zn~K$(gLmh&r7ko!vu>9FeeKIo$7N9ztAKsC!DC5Jkbs1`?c5&DePkjo&g@-dKg*d1aIoFb`7b7 z1*MnjVLb;tgS#19R8fKJi4U)sFJ7~nUVVFl`#RKPpjP`*>*r^A(u&eQ>?ee&v_sg= z)>idD7i44Mwtgp+7u_7U)Bqhl2KIt`Wd_WkZ$Wtjn0FJtQ>w$vM;il0#dE-{_`@sw z9T?@($KZ?}J)N$8h}^<^uUgWqP7ur{uw`fY5osvX5yA26r&rxB{W>LtR&KhPVT)#K}@9L`VWzdYo^r6@Q*?vuw5dCBYCRA;M^f0{sB2ci zT$;hlYwMaA;NE*R*JWSrNNw%xeC@tV{w^zGyxA|OdB)doaXNjpRuYF)*9FTsp#h(> zD3yaZmDin;G41+NkCX$ewYg?;{WqG)67qwgD>>Ts;FgTEn>iOHS95u~Jl1_?;6C#W z73zY&F?!auzjAlw-4(How>cjAv;B&`A-BL#{}K!XvYJ25KVU?wIZD>``Hyl!TQQ{GLvZanU(FbRij#kpUD;17;ar=UA`gKY7NI;zdS$qD|!A;V_z9o z~1_^0tRJudD3>IC|4Js*0cZw`JmXgvcDh<*dA|b7GbLZzicb{|b zJ?DAuueIaj%lXcC#u#IEj^oQE?I^lazgo~DxO(2xeiUwH_zr}Mf(#sI?vyHCc(t0Q zoMvZ6^kAX*v^2w_(szto&3UG4Zv=w2 zB~1V%ec~2cV(&a%6$+WttLrrGo}L9U6P&fbpL-y)W3!4vt-T53 zC6Xi&al4y;bGi6uB6~`@_mTmRvo|K{RkzP(E2d~$F_dMh=G9Ym{-x*BLbyrV)@P`e z;bAg>AAgdV=rUTq_IGkRhL%DgMOV*3D5K%`K;mJZ7_zP@2 z$apMJ*CGYf$0xu9bYtX5e#S&JQvmD{8b*YQpjOP-Fu^x}thw}G29~$3g;C*uV zgOODdpSyA!$dtT}g?!Tak9~k)GkMNC3Dr#*kQVFPM4HWja}zmOF8x;&2r{EqmhfU+ zSdB7f5ZbnEh`v1a+UV>e0nS+Mha;saBl9ah2EI57VPo=1+VttS#8wT+#;vYC8G*I| zEqD0S2>5bm^l5RTL|}&Ci@&5eATuA!l>G_qJU;u2IBK3WKbyfQcjAVMW(> z`R@=zdG)jKTI~zy%e8?gZ>5$M8w6ZUXr+D1SPGAt>u3ED!Zluy5r(alYXfZs$$>7|k)c4% z1@j7P0i3pY_=PJfD2l*VbCu3n7%nr!N;wryjXo^ci-8 z$pRCS_Z}bWeH7m@K4zG$IG zzE)P`=T%(TkGRnb071cj!J82+c~L#oBtC&ycmE1Zi9H9HcNyq?B9%*}A@8a0#vea5 zfRK8N{?~qzT=9$McPN^hF20<5JQ*tyTXEneVct_aclG>Ko2@2{6%6|4y;QKse7j4| zZE_CE(7}*F=a8SxgE5?dLZS(a!KE@dwXJk2tn2J_CKL(wptJ@gbv%Qys)Y_n#p&w} zJqfLpC*+c}xf^bXt*KvK$-kyt{T{SNBx6P$*z_VLa$(C~yc zYwc}guU+4xR<0|2%mfKyUbbTGLupLNN+6Fb7eq{B9x*9;buh|u_k$z)%9az4#U7Yv z@aA=z$k|~0!EYSU55KF|tMnBKLye(M@h*x*LH3M6^^KSB3pyX~%$}9o$iLDN#5JD( zR!x`MmIv`Fw?G8|;W0iaB?X_CPg|Pz29@=Fta*A7Q!|FwP}9Q#hIR^liee0(7` zvFRY{L1T%BA1Ws4uX(&zV>yKOircK5s2Dk-7$x|~PjW8a?Z>_?6h^WLV9zjr1CDI$ z^gA;%A4jK|Z*=f|XNsmU?huzWCD`_>mg|T`$GwKfZ>Gmlmz2a)P8TEa_4P$&5|57? z5?s@-azH=9zATjUqOf-f0M{j$_V`)qwM%jjXGWX-Ph*;@NQnd12Gg5LGLPjpUV2nd zM|Wy|FE>@oUP0tXM=wMG`Pm;)(J#=(N-HY9oKp=!09Ocp3?S*x>gwv=YbnHW2wk}& zbYn=I?A*3O+S;%4eLiYd^79&e^9un&sdm>}3P$(dNqvy)8de_sP^UfQm zi;VdotGNaTARe$yH6fHjjO=ATPN;XhI79Fm*7t*VNTu*0NTXiAV^8CAw59C@+=_2H ze7Z?BW`1?iexOp}KQyxkcu+<=i-qdcJB(q1HODj*##pgX_W3o2j$44_tzXt60FD%| z#V}%>?X-W3xAlOK!%WG-z_K5uU1Ik^CfOcIq6)MK08F95Ckw1jPF#9@h;W>11{!{~ z#`U#%C4(c>T0Zw~Axrl{cvvIGCjL)Xu~l%0it2`l0mUEN(erM378Z8|-z@;L_Oh;6 zCZ4Q|i;KOzY`Ui6+=7!3-dQGr49gx2{#AI|O~N-*j9B~a+hq5%#(5q_Q|Wb-mf}L#`V2S@(szkSZDo+fN-m8bLH{dJ z$eA_pBH1ZUPAn!gpW65EGD!R9Z_9E5M2Cus>RmR7=Bq1cfqDdUSpdtWEmy^`9?73uaB-D9&g&*YPAvjkmB!*uWPK24lQmHkf%ef`s_v(HBdai_ zDxYQiohf>1IzfyF20>kpEa#E;y8^2m(m)buxeE{zmt9~#(vh8*n7HGDr|mX(E$#=f3YcznQr7u_O85^K;q2maQCINTdaK&q^_N ztm)QV7h8OAT1sTt7;qZF*70i6E{Lk}AJ80q+?T6%$LUvuyt^p6^~y|Jyb8g!8!(?0 ziJ2fh{WVs_qns+w#YmXQbdm{@T_3-U0BIYbuo?LhT`V6#;kZ-{6w}%R{8VdqcW_=M zc=EE`$=70|zSb~-*RX|coI+N*b()%GM7BzL@iz&88m33hY38X*^;5MRB23zkxnQEX z#trrPazMCC?6C7Hr7QQ1=Z7UeF_a2HrIy}wfJ2vMrhk+aepv8VY%=HL?c*B&Oka2L zoPVm0TeoecN=&FYig-7^ zbc#VPgOQs(mwSC3ooZcWoLh#ulC9kZ`cL0s{_UI@cxg zA6J0Z+&8|cFBdIZL|Xh9_~bl#mX)IfBFKfkq>D@rD%4Kteuj0N zW52t9AGWD7U)i2oMzr^D^iF;*)O;!tL1{JR)3VUz`v#HrYbbPOAj3ei&AZY>-GG^+ zR&J@;Kc=i=84bW88~E^g;GkZ6e{7VINDxrX#-DolCJeDr!JTkJw3fF z>?cZ(13?fhzFLoHkAlM6X>@Pq{50M4QdwOW70I5(^X9k!+T**l2inzQ0@EWKA*7bh z^naemA3SZlOeQQYznv279AjbE9W6>O@2105S$qS9l8CHREi}#Ty>UKbn<8;a*1$7> zZE5l#Xz)mX^#+ZD?g^ZQmH%yC?O5>>GxYLv<~{eN2FK<_E#{kHx`muJVSZBZzc zdLJ&m8OUtJ0B*YlcHnmJRM$QjJz~*=Y02_vc{}WhXV(GH*Sryr|dZok0WqaHeCGp zLoMvy>mPB6nxPSo0mcEN)c{~TW@5FCB4H+`td35CVlqE_gK_n*5aHO2=6Zjt{M}&e zMnnzUVxwoF_$Btt`eAyj9Vy{f4y=*hw@phnrTj?eODNPwI$R9Yz+2zkePIG>%^b5y zWj~-I5}lB6T_Xs^LhXO(>CK}5a#zmjy&f@nrLk8pQE3}SOl0y;DL%LRAFaI^?9qHq zT!?Hy)Lrfp8Qg{5LRO0!9281KP(of_J_-gZn(VnjA=TvAONr-os8-!;*+1*F^I^hs zG+Fy+WTSjTHhvaUP|AwYGPI^+iLj9|QLJkx@3php#~ID0qm^|1(!(!W80%9lQru*C z&M|!D9t!10du!6n!F6Rsv;JVUhK(8!ZSMiG&(L68VcuhPt(<%@sg6S;AeB_ zVf}``ldpT?EB#Be=fZ>pa%m<{u$dy=bvU0c!wX=DS`)t;4g*^Ech)$;w4={bd#S>X zr?>~io{md3b)Gv(RyR{c+4IAOFHDs zs`agXlBHq!&^1l1$l>d3;?fQI`7>9?{~D&$-TJe#0Mh356|f$nfsS-}+mobzWj@Q8 zKl{QgEG&CjHYkzoImXFaw>N|@yc1dx?3t2X`t;;0Lw574bl2u~V^_Jy z4*4v%sD3du7U$h^HPqF3et0xi_Y%xS>KBR`P0jSe7;LOpWsqoa1GZ7f}QZsY;y;m9#yT zMQ$S{pK^&sKW?hI(MF7dF0)~n{+8>B@L1W?SXK8A-sq}oIRgB1Zw84_s5fsS&s<=B z*!@0)8{NX6So0W*$(|iNcZ-TS+_{L;J4q|*mqzi%23$nd{#=CLsi-K+mRik&nXI+S z*1REljk)Ra8W2X+Ees2>AfzT<6aNGu?I(aGd-c4 z73C~oQ`t6W)>H_3SBF^lbsTju3?DU*sp4_jnc9)C%zu26^C*{$?hAQ_4w1|X=hkZH zak&2oo_fK;AnQ}~D||yBQ$3m+EJTt{PjXFKuF|4G=s8`7tU!e;hVA-5+SEC)$I$OD z@MhF3^BK~yf;60L6Rb^Lj0xP-)RCiln(-$~>XQbea2RSL0EOTD!ucQ*-6xPdvv$-S zRuae8mhi*rbwS+YWO!mX7I+TQuvEDSxzNhy9A8TN7|n|W-1yf$_=@h0Ag+>A#t&~s z0&J=Fq>0MieZfUAI>Ng3)REV08e#d&rvf^R+od34@O6(q>QdU{B)L6mKU6B&7!s5j zE+@Fv+$(xRuEXK$un(Ln;2aKdM1VKbWAxZ(W?f?EFKSvf22b7R0;L_CkIq{Qz(=|l zjKY^?;^)#eIoVA)6^($Qk&>6U_dlzKSprX%IVws+eYWu* zV{BRY0LHksMCV#jz(QoKuYK|{fO({qqqD23AbaP#?OWMIO^X2rMv{emN?v3<|&EOoy2ThW&P4c+?*+^a_-yU6aIGGVTDim@%|;64Aq>KW3k2HSH{;F z4phDgw%VENNlCld(P+vCefYa>r`+MP%qw?1q`$R?N1;c{`YS3^+$Cb z3(-W^kJ5r2CUMvKcYhq?%Yckq^eV`m(qwtNGxCAv# zPwMrJj-CdkD9Q4ppB>jbtZA0d9xFRzbZBb$;K21j9yzzz^a`rjryH3jgMS>uN;qD# zwj$=ZRr2+pDB!mbT=2YSSbFr+3vXwF*kvfu&aI^JY66n2np>?kf7hj|+a;P$nW+l0 zaSP%r5tZ>PabIE{XlQXa8&g$P6pwWmXX8awKxBg21IwjqDPi057{BShI@;kt&-h7g zqCtv}th@YK3@6#lz!vxoi64@Fc;TggrYxc>Jhvt^!b(hkWTBm~vKypujt>vcMNDxi zrzk~`HcT>7A62QaEqUoy5#L>x+&R;Ob=UmUf2#pLH7foR4)GX=AcN*y@z5=eIP2(y z-3^PHBX94S)$MWy`cEtY2FP;o_*+)l3D9o%F2j;e^&iw z8Qc= zrT8tT%;RvWI7|$D-2WrK6gRG5M9xHxL(IVps}DOs88cr$PTSN;|92O_VQ)mc#qXrZvdAXE%%de7ptH| z#6UwM0h{~ED*vBzu%equ(y{Rq*rU~RVc=V%@+x^_yVmSv?BQej-3y_jzno`KRf7LY z?C|$}mc=M_b#y1SCwj5R#(`06IOg>3UUS5D`D{k#Wp&zM!!yY3keb!d_T;EgmvTX} zO)oWUHyYlSu$OFd&|Ml8hJb*w<(XBBo+`u_>2Ug_Qt}Yc2qq_-X zwe_DBVVOb|6eC#_Dq6}ld?LRlA9JC<;vUU}tP;%BoES2s-gw~j*Bzryw@%)ezd$MG zrNgS3PY>10I8c=`)w6@%z;bx}`||rH6KM3PKhTY12<1Pd zgU!DsqxVnPx=%+z>(B8UvR}VHtSqL6`<~)3BBeQf`gG#duX|z#%RIeLcwuytjIO=? zI~emJndY@FH{9Uc;2#vEpjJAYejR+x*W&1~l5d*#(TZ5vs<+Y!`h$qA+6!PWL^xQ& zCW=o$;LHN|7rTdW@MQdAQc|pKA?Td5^YW^d+bhC}t3rtQ;2#zfUIRsy#i!sHw0K%TE@<+JLEIi zv5zurXstw-zQ_doqINQlYH$2|QLG7( zpli3|??ZKSTzX1*m_YkUF=&FNj$hqKGgtM~7s-VOy5&n)&xSb%!ETV+`WPHQ&?N z1%Mz0#dJn4u92E`Z0s#IFug zN7g_s4CAJB$Y~*kpgOwkF1sp0`Z znQ%gMgTMXN-98%GdHmNDf;}dw(9w^8{99#vd6>?w;i%%unXG%Kc)%?M_u-eDn7s;@ z>x0>@a6(kAh=d1}Rncn`pkxZNm;qIJFp6TFOK$0CXEGingQS5oO_!sb8r8;8qj-%? zheq4bIrhpKcFVX(2ZmF#{=K(_>r&MADK2_c8idgN^Gr7{M3j*-Gx28az=H26y4~Y{ z0g{&(a3E-Xl@~dmY^);QKsbnE&UcJB|I$cYPU8l^dxzMuzP>p^JyoCfUU@8N@C@y2 z{W%co2d4%>1ge$*5(_+8k4J?7Y<*ed5jk<)Dd2F@d8_@lkeP>Z+t$+_Pvp1&r(jA0 zi{CuUN!8i$=KY;dh`ks6V>mRl^!iy;9wJ-u%qbl8n-^mA`p-v`WW>d=-;hR}I%%w!6E#=8WOTtV`py8 zr)>*<@K*NWIv26t0;ryjNipyJnL*mWmt9YR?TRx#^v4s6^ZW*70L%lAZA=f!Siqnh zBNw%b*ak@|IY~+X(E(id`+t{yO}Ed)>%SM;8TkRHA92KdLd0IPpgcfY&gK!Z9>OmP zM(|bR7a=(r*}`hS1l7NP^1qFo7L0u3v1gbyy#?}+|IMqp$_?z@cxY55HvAU*cXj-S z2g0rg{2q&Db3y(#B9Mw6`IY@2-rIk^d?S<3{`i1Pr-Sd_`FjZP%vpd+R)_K2c`(gR zo&nNHJnTHOva-Qtw&Qw$ds3HJd-Dsl@m_B(_A7k;{8@3-e6()8 z^%jS-xo{W2n_b9mKlFoMP)C^|bfQdv?M!^*#*%vYwj<|JO7CU~?7Tcpi9XUuJtc8q zyw<%mPy4MTzJ}}fH+LQ2=!;wG8X9O&^ieO+V#g;Wj6;?QY^$znZITnCDx>$?HcNTo z%e(|pL@@=}cR67bPvqEHsho_1wpY@2fqrFi?fzmx;()N*?>C!klg3bpWzWdS=&5nF z8wQMjhckYdxW2l26t87pbDuXXRvwYgKl6zz@kvRnJf`gtAUcw`(NuzNlLzg9S-^d9 zq@<@`tNjdSQv~&1UFg?Tf}DUMm41y&(NzixYd&C_BV$>%`ZQE@Vd||GK(@Hq%s%>o z*mgq$pVzKE!m@q`Ts^B*PW_5P)TC1P?{U{jU6_rj1vuX#7L9BjfKilC`5r#Cf{~qJ zLS?CDfc8z-tah>}fPiJdu}jn}EEdoTy*=UWKrRtn?Dko$gyv2!2UPwmg`5{nj>Un> zpeTO7E%ugKf1XB@2N-b?Y>GM%hrCbXx3OfPr~lCBJsrZ-?r^-3ev=XusIrPEfSHKcqVeU=U zc9j*Ct1L38;%+<(LXj!Mf_$p?j=L*zb>6@22aL`w1_p*U4|sV!M3$mj-~)fDi@E)N zsNu6dBj=dD?bra^(ItQf&7HvQo5dOb25w8kRbt}DE8{hTY856e#2Uu94@aTPQ3%dt z322B;F$?%^00U|daddESD2@*awGtQA-G>|I5piu#;L5JIb=0@EE`;gIx{m5|)>9v! z&0)sGTw>2i?H_{vy$HPHR~hN0VTL zIzU``g)zsLVTT!_?$t_}Z{M~bK@Y|BTnsG8EK%Ka@R-QdRX-qYoZYvBQ zkeDE5ut2-3dM?R|V#nBXg|GA#@FlAH4giFsROM*rsSM9**Unf+_Sol+_3WD5$GfX8uE+>iTu_Zy|su8c@qV z!D?gE$jnv>;XTf+fxa}y@By=B9EYK0MQNBF7CKO5#sxdDKz*7L+($YcofrFB4k@`! zf(1E1XoV9Rx$nt*i#86N2=P|tMK|w=Ko35!OYUHU^JpGZf$cPwGw$*cscid_^#|Da z2@N1|=om9RD(VdtrLR|~mzI{?4CdtujT^+;wnwWRbKsc+v)g2#zaNDJ1E5`KI#+EH z__J%{%a<<+6;{I~St5Hg(dzjRSR9o#YFupKUsTT999*Vx{{U2*)`>b#G|Z1n6!Y2r z{b-{SfOlk6y(NIPt?KK@Pje^^4NafIJ}Xe(g;=@(>DoHLDoNh1T`I0I7bePymflL} z#bfM97KlgiV*UjCPrkx9dL_y*(Y2=;6+-Uo7`cZ}28%#EE|BEIW_om$E+Yp)>bYWZ z5oZv&12n|+{5;cD)(6eMGc%M~D@VI*sq0BD1cvGKbam<5R!JdbVzemOW2xi3a<31y z_hw`*l(kkC*bU!Y2G zi)0VqUv65d*6nvRy0*^5^U72hbgU03be8~p#1s+|vi|LcU4peatkVMB(rg3geoY6& zvyc=3r>PZYP%FhX2T^$hZ*S0&3Joj^?Yk+9GdY$j&WMX%E6+Iwjbj0_U)I$feAioF zZMZ&G9wpkl&y#6^Jpe*koDK2-j{NL&C27&cI`2Q5h-)Qjs`K<)@XH;6q$Ad>#$dbu?dyG@UIDxa;+n<7-n&FOTn2X?<7j9=;i+$_ zUCQG*)C418!CbN%5Tg(jr9Gquan zci^Jh_Geyno@ZO|by{?quxs&m#&n~+dq@u-6f$$5BTg03E|X%3OBEz*bwZj`2MhkQ_gc!3OU(djO{xBHI%UJpSv4u zn)wg7?zi=(cMZ++3ek6pX`RtvPk{=mk{yv zJ835a`gVyiFDHEtiZZPMS4Y-kX-RkpYKGCDFYlSbcQnpP+A=(1Bt(3 z@z2O!^z`(o3TNV*L4lq%iFqrstenBS=Pz8~X?j(lCsZ9qR~ z&f$7e5A|NJ4_s0^(2=aJ^YuXg1^cBV2zNE&e|GVNH^@-Ae-WAUQqvJtAs`?qhkC^i z=39y@GJZ2??i$rXpIyAur**;o2nhFUsEDG0K%8tPh{+M0Us+;YIQsy4Z|&Z71H|j` zMmBmxn-DG~TjBny7(D1CrNOL-9E?2iNIg^cCZi(V=!(|pTAJtT)4t3jk;VdSy(JQh z8n~&B0<6Giu6|;fg!71afW*44{S4E7{j+s3X36S9}loaiIA=nzOqX~|J-V~l?c zCZ}l+ zBNtr=izI0h$~I$^W~G`Cx=gbf;n|dFnpGM%vMm! zne&h$pDKiaN40_)5}tDrF-BerFDMCKwZ?PWy>*?ZV$g?(1c~+v)U)Inb9+K(*a2MQ zH&Aabr>Iy4r7)ZRT(!o-w{JzA_fMx#*~}+u6&qMQrxn#IJKAiMP6=i_LbvaE5 zJ~(DBy(C!Z8mP|l<#j?7O>Ua*>XN^JH#7-s6# z9f$BL>|$FApH-YDOs4Ji@i>*8UbnB{;;|oEc}9I-;e(NZp=Vl!-K4Uk;q#xIIsfd_ zlc4BT1ggJDPo!`&;h*(kk7F%cna@1=;F*A=)hM@9QM|~n1o^4|&pZP;`oAQ3NT&7o zkC`PP_R1-<8dQcDo&(f3OQ2AsQSbKWvBf}PA|4pkI+$d*c|3HV6K_l@K{E0rEECFf z&@MRj@xo;qoR3m4>LGdz=-cXmw!lAsJ_^FA=8w;CmE1k#9u6o}3<-#4dPcD8J8bljFCb*|iA5VG-c|H6gb# zX^yywq#+w^Td=6u)4jJ7y*9gLdLYo0IRXiMWJ8|?lwFMHYE_e3ah;4=XG$bsNL!$< zy_D_x;=*ZJAkszx8apXX49fOWkkY!p`P2z5a120^)}cHk55+plaMATtXE%0X zgH)nKOmd?V5=oA>*8IcADZ;LuMRlrqd3jL@3Fd(pKYN>>MKy~C4&g{p0hmko@slTL zsC96Bie%L;PO1I#?FOT4BCkahOATz^SrA9hod~rn&&v#Lcp|~K#zYBPL8I8&=9YGG zjO}=}9IOHi#0;inpx9Q^wzTk_)7$95gP1XJSL)e{WEPY4bwB?gt^2Fm29E-NTi6Zy z;ozE_93PQQMAyed>XG097{Dl~EX}%k{#p{>{C6AYPcuIQ7+sZTGENJatN zqEeOEai{Q_iOx5Ou`w`vkMUiBYR2=!HQ$qH2oXBj4glli(${N5F>ezy?v>~7k4FZL zYXAx*3=$F%#WzuTw#Nv=fSHhCQ*ulkv)aAIK`bXgJ2CEx#`>eu zhPJhTq>-E!|6GA)foa7bP?jtmH?1xPXA_5jzS|XgSBmQ*fS}y2Dv+~ASL4#$4*OFU zFtSY3%Z-gkS-m5W{G?RG+jk@kLOyH?AO#!O>!FBS8jd8x_wL{Ca6uww!$mKMDZ-A_ zjJ1Pu7saL(N3}=qwmGMNKUsRp4=w@B@@Zbd##UK972SxBhlqPCg&1lA%HF8v7l7bQ zCUIV2X>HfkHbheyf3VM}!ZO7OsaQ;Z30a4eCzHsCc0cgoF8SOM!aMzv@@6RnTYmy!i)=E# z4F%JxM^ph*w-7xSBjdN)xR;@!(N@J((tutts|fF$?WxkKZm;bj@$@EwGIuh7ksbgQ zo*DR#w>KJ)0-mw=R%~9*#@FJ8V~?ukk~YN~=BiLNCzPSd@9IU=00!8aY#qn7MijDf z0?)95U>(ohsI)##op&EoFiUctF1*d4pBH37O<9~bx(0x__>UhYWZp#e?nD|2n@D?d z)IcUOG84j-cKhHt;pP5}%uc6y(l4i1D{Nk~YF&R=EKf$*Xf$~6>QP`W_Jjlw&$?!56^-K2POVI7f~7Fw)#i7E z)oGneLf}G6l_7#~%o6|>Mrk|f?2V753pwXk`9i^lZ8E*$^+0!pLj|}r#Zg`+C&R5O zP)dx-RIF^Mcu^5lpH@~3FcX$j+qW>vrIJD@1?g)WJPr7H3{%6do((Ecw?&q=_x65** z=0z>Kp7R`?9#7oHGpAfheV2|ESegBvmJK=$EnMyLo+O)lBaV+sgV<+d2#YNHawC7% zr(t31ns;a<5J)%?#=h9T`0G*`pVdG@H7Gzv!92k>C4uo7w|YXBKC0;2o(zMLOsHz% z#t#lq-9b0_)(lRA$M1Zn#Q9}HOz4Vgkii^MA6m$Gzpy=cFhV7?$sw>tSK!k8@~FPN zQh|rfRN7d7@+V>o>mwrtd5sgxg9A5TdKj3iJlJpZs(}ap`7SY?3`NBI5D&9woN=XS*3_ROotcG?gk-Za_R=$209(iW==71G>+ymo7$A72TUgTa+U}Aj#YP6$$2_GaBw(5H-fVUMG6A z*;@kh0IQ&?tHfXZ=Vz1g?|u0NjS{Qr(`Y8`=8l8r3oBVY$&8!dUpabYJgEze-_BjI zvU0%l79T!sw0)d!lPmgcc*SfayYnE?&{RC_#(oMSM}?@pa6Q|5{D-QAc3Tr+3>aUz?h0`bv%O z@9?a=?m23AA2lb+A-vgFC0mts@VdAO7=#q)>iVR-%c%92$exdzirtv-TuU?_-SXGz z)v{7~y-jY>YMt6FW;Y?y#JN%T4&{I&x7V9)LrbYOvl${X=gn5+v+=8x`mR|MFbopxHo`unY`8ipyOn( z%U5w7MIg&7_+nde=u?PTc3vjP=UPXR{JJEqv{7`|)MKO+7NHTfyvmgGaj4M}%TdL^ zg)Pf!D*`QN$}&<(m9$C_QfYOTO-w7ZI&Vk$IEU%i2wld8rIGKEGZfKRDf`e?>bRm+ zgEqN1(e+F#!`)6JNskRZ7dA}{2S*7B9l6ggAeG{k%&x%<@1!^PaqoME=_pUi*e)Cw ztd)XH+1$}WiXg#fr^%j=jcGZUgMc$))kd%hdsx_IGTMK?GA^X8ke<1bP-ep$Q0Xno zF)l{_Ytd?9W3z6tzOb#oY|bUgZUyQhMze8@sP2^v$0ny6i^o|lavV+stf*$KFo$s* z6nDALkw>(VzJ~5uJUks}`E&2^#kzP@$!)BpNLPdV(gIPOB7hx7Pget7U;reHBW`xM zYFXq%R_Is&-4ZP*gLM>vV%Y#Bpym)D^k+?gZ?{zsURnYn(a-x(qvFAIcD}5y=iuRy z38UZ&hul=SqML8f;7L!g@Mw)IR_?VC^jr!IsvHLB!bPTKnO0Kf^errxVLo+|GtyEi zg37*Gb#ap3KD@JcP_9rK=SH_C@ODE|@g54|HjBS1`eIg~VDTP?{&g7JG{QK`Dz+uK zkqQudGEv8jvS0waiXQjg8P@`~W8u`)!&_`@?bXm8DT2W85X%6>V<8fGR~Hm;j!n6E3=)fb6^R z9+O@hultJ)g*7;NdHX=5F&0`PBJo0qtCSlo!sd-4x$6Lg{058WNp|Dfy`?+OX26wt z2SWl)j8rt5mQGeSmoR&0B1pydRkm+lL7`Uvfc$71^xbSclzH3q9=%m`8w% z;ibE~i<=lPujBu`g2x$d%lm?*_9M6ozKgQ48wLic`^^vL+za~G7#LsQ!{lZ4UZ-!& z;=aB=I*zmBHB1e8Gv~W~53d!*i;`arx%Kg52ooOrCh_}QIXd6(pz=jHP`=x(WOy0( zo;>lXns?**8IWd8Mmk|Fl6JgO%_A<5lF}+5E^fQsy0UiQ(@H1*&jVXL5tHQRG2})N z^ZuWQxn=E^1!1~*M zFSTf=D|=es^;tW2XVlQ5^8WB+<@b@1KeA;*wJM)0xUY?Tu+gOagImie_s(*v#*W>h zP2wFPbwRo09y6hFbfs1IpR>dDlhc)~SQ4m_4{pzyb2hrmZm}zJEosT1%!JXR&2Nq3 zZME&-^WWdzHn{u_Im>4h@;~3KU+dy_o@@ES+J1SIc7_^SKU|wvPZL(Jb~LOor(GG$ zQmT73iF%cGa9!s#o4OS)*LE?GAs*J|e_p(wgqr*FBiwZl4eN~NmZ$7!@>ti(k;gAJ zYaRpNz$lmZwr-IQi=guy&0=?Sw-UWSwup#`HH09B9TrZ`tE(bVX+qVd|trz?e? zit1x1E^&bm$MQhNQ^``~k8p}ghjAW3rn^%|TJt+dVsjn<}4DwYHRN@@9 zpP&vHTASTgqI?G8*rM~jSUOYrtZ?{k2TD_+q&D~An?;5dnfYFwYsP1DE3;nnKfo;; ziT;gx&d&UdOk4#8b}>?2qaI05V<)moNG+@;*nsfX*+>8R6N1WDwVG*g1!1oIrSt+m zl`V@k)9j_`kGC{V>g($(x1X3d^;;$AXzZzT(V90K8yk8si=N(ZC;0dB;6$y%_^%2J z>HCii2+GULi;>xSyMZUa!|?F%l!qo6`iT^_=UPXv6ciL#rf9kw8V5iugqzgbcg)Vx z8dq6spYvSG{289B-l!H-h+&Pa8r0E%38loG@|ztg756MDn)kL_BN6ZX$*W)V>uj5i zany#OzRRYbCnFI20S~5p6R09rhYNGnQuyt&^36`);FIYQW*9ZO7ZpR}s^VT`X8G;m zfBW`Ltz6e$*rv6rqH2(|!EE>kFoC^&Jd2EM*2-=S* zCltLB@jcooe&=z0b;&xOn{ggZC#kbi;72JYE}r{pvMO29S6FB5?@;dFiOO6~dSlIw zklUjbfnM8l>R-(4gdcT9P+Gm@JgZ_A?b;mWHEVj~HRrA5*f%i1qj7!-qVAJTUt;k) z-6OeyPTuwlQq4>WSPfVH1voc{eFu(G*5)L+2euQ_3h`O$?j-}%t8EV`XP2soE+1Bort4pAPNA(IB3=p5v zur5zl+ZJ!!xzC>6t5a?FNew{UkCd@+#zflA2!vlYj@ zL&76NgYle`arwo4i>uj2H(QUWzEkw#Vy?X+snx)?{OJa)Kjf8eIp0)ejKll(_P}%x zWSjl`a97shiNBMzG^6ed= z=L1U?d9!G}HMd)jGsTDsBmypsGS=Z-Hr}T{TX(ytc^%=rmqbgFeI9bX#wIJu;V~y_ zv*#^qQ1~%xJrwa_VYp)Mf3taNcuOKr*DKmbC_+qS zmRRMSVjvSIi_y}0qph!W&%Y(`?%0nO7Y%%Z|4z8PLriQavb4Y4uPgFm>fE2w?m0V! zcH_fd-gbB#kBCW?ncdS2Bi)n-j|@IrSfQQKf7mm0l6Wo3c@p8d5Lq=pxNg1gv6Ba8 z!et^a5*-s>FeB4DuiRb<0;;GA5tt?Bp|aU7m=z!1I1&JR%wq9TE{w>3iJyFLiQS?L zzMp|2ssP!?AaJ-qoqE(%`^e$@dy?Xor2+#dyu85(Kf69Cq`gg5;Fm?fDa(LQ-k9zH zBR;8(8C;G-giz;-^g!9$ut?{}mzI{SR3V3AhDL>O@?yJSY_%$G?+nwu+id5LL<-?8 zkclSm{Q@6)_)LCF9`cFbUcN=Mr=v$4ugC>Fm#HJ$FAD9((Ef)d%#W|vn_Gmwimve) zg4%>5YqZ4BYA{oB?VWbBb^@;>|OHo(`2jB#_?LnDjZjcmkHNx|K zDynE3Iby0}sj;ELd&aS84;V%0;Qa{xIPe6$Air6!VN|Z)i=~!WYNN(XB$Ra}F+EGa z#K7N<(rL0vL2|rEmu<)fP2)6HnsVn1Z5qE>YErA>PmVrg=^mQc`Q54i7$Ty!7aC-$ zxpawiZ(NP17J&|$E2s?cHkSAF*O_jo3ECmD?TRmUm{+onm^(qP*CT9kRdP*PSy?G8 zA40+m7|&VG*|J>m1C~^U^-Rdd2y5+!>0SSEvF@A6*OM$&~5 zbFdLeP$S3V($^Qc0~FBm-9S;nW3L?3KC}|gWYk46bGP1HuICj!6#o4emuk5c)0M5+ zBP+%%urSEWSn8E%=_{k!d$HSu_yOo>CcpW8Zb6e?>uh~Dk_!j8~_v} zE%bpGd;{(&tl)0={LRhDAuZhp)lh%G!lKm@gLaj3h*wORz^KRpSzC>;xadC0~ zX3tH6;ee}C+qh|RMn*>bBO#8Ps=W@tP1Z9Lu!K>;NzdmtOwpj)b)~SKGY?Y>*<0^P zt{~%-0yn2>vDatX{F_sGZR|?}F3;?nqNAgw9zXu|_;PKuq__n<4&L5*@KN_gl_^Lq zhm%z{;ks8G+*XU1h)ZRj%~oU1Xe{rv^^k?R;UpKV5ucCZFt-ZWFKPDWe zmkubM%e|NdS$y%;%E9WekC&I1|59(lqM9=*?*r1!YP4AY-6CuEujGvqi3myWJ^5=8 zhuGWOV`WdzW*XhrRTDT9EHmq_K`{&!c~5+KhQI7QU8mk(VqEc0je!i885T8HAM9dJ;c>ZN+YMv#7^fd@4mCH4`hVVn!%(cR+mm z^C1n0Mzmj576=n|?ab-3zuR@Dfy=G1bS z`0^?BcqJ>4HF>3=}@b^?WMCAN#7&U z;Tq~MCn9rQS+r1TiPoCdwrbd+Z^F-20WmUYCNv6 z9n7@P#J)_G@;`(4M8;+7IHfO11SX{Q8A98s1Yc&?72S@2hvv_|L_#fF%7q=LAnf6f z49h<|d~m&&94|8WnpyTSZQymJbMKO(c3U6%ykGAg)BH0y)=ZIZA@)jpd`i)}Lh!BI zBVMNVfo!b#ey4l*QTu&T>mVF-xAl9)E9ptlCh0TlUrTV5D|{{=8wTB%#zO_1An2JP z3qy13AIR87_MBS}6Oo5@(&hche9RqE9_wS=9ebpodn8Uj2{mHss0_$L+xCp(2lbwB zOIBjo{RyX+2{>J%@K^lC>Up5!+q z<4HGtpkSkv9jR4uS(#p?s=g20&R;Cg$EY}R(RXJd*gk-zA1nW(QR;7H0Wv!=hJKDb zXt1CIB7lekNynmIk;jz>eJ5Bkq@2)H^C^ix_6kx^jI5QS@5)1_3s%M!+|j8+ZG27E z`_z8NE}SN7jB;LYpS}#dg$#AXm~V5@gk(0i%@UfD>hpeCEFC#!st0y-q_g}aD!6Dl z9QB5&h!t;lbGDg6`tl%dikKto#7Y_DS#eqEyLLp@#mM)9)MO|n#neg2(nO^dXndr9 zu&I0HO=bz+jy*nax|Db$I2cHYCkW_5_MtP0zU8jAaJV=(6NWtbfDHhaEh18m!Q|1+CmDCEmOS*iH z7tixx;esUUYct9NPzLqvJntkyL*Ik~{t9>ivI->#quZmca-0_SaO&~UO1T>%HxgUKvjGnIZmJn)E&PI<>|U4)Lj?X zeolb&W1nxW3V;&l(Fk!siSwO5`c)tgM@DL&iZ6F%btIYLnva&4OaJ^a;#K2kfPUR~ zs2CM@@l?EP&zS#ElKGrnjn+_i-mO4tkT-6|O-sCue|E%H#bGb`A!yqVxsHV>@v$H( zOyuD%_=)}sFA|2;BSW%TU0I8OgNfb zdY(4X63_2}l~?PkX`eZx+-6VN&f~UK?fT?n-jVU6{Uq8nbc_@aN*3 zu}QbhX^7A$qj+V9aqY3K-(uXuO@Bz@u5y}FxyHbUgz)%wcb~w?6^q{a6C5P~6#XaD z&R(_jdUN>SL7yo5`s2T}Zk>wEm}w}01P5;v&djK`ORnaAI-HJ^QbSp&onf?cWj`=^ zjRdt~ETwohv%bV9(fGYDs`6(Vs&ah6Wf< zuVTMSHkz+YJ=>}bH$Oak#)G_!8XKTp6(^S_fn8jx4K&6f9Q<|G`8?%3;xEtRq+6xb_fjhiiXxJK% ztO{?>t+ln|j+vo1mi)XyQ*M#h4b-J)qXL?SJUu;SX+g-bbbtmUT^6*02q!MsgO z#m!ehcowwfE`>cr{wd6Vq3Jvy3HMvtXYDpj-77p0-})B59_K1N7!ia?$^r|)Es_#W zV6zDD_?5~>NqoYpe~r>GN}W$BMkIw>s06|U6m%L?hTCOBaZ=Rec+M^Tb`kD11*u=` z+KVB!@qgz7&ebJVhpAwN_G^rn=@CgntRlB|yV+wb(wA~rL~wV{>5WZP`iy!iI^ePi z1rmu2i(@_N(Sk!UY4ay&Q{k&8F#(h)8V%}DWb(}4g1j-MGlCw%ce|nza9#F(vHW>e zdjyrKojD}Sxbe3DtGX-t5=QtDdPdKpFz|qJ)lofC3DRHmXFE&L-a0a^2}zhgCL&Lr z^jCjCmHVyB1vTbrrQ8`LHAiTk&x20YIJF5s!69uU5T2Iv^0I{V5eRXI@)-W%Gntnn zKXh`rSFZZ(k^TMwcC}AX8(Q|OwbZJPnSO0~^O?d@34Dz_y|s|Wj_`x92mD*HuJh83 zTdHDaZ7dAxvJhl{i(4p>5FIxgKM2js6CijuggWu#yqhUS@@}EHRHIVTnUR<9Bv)>9(kse!d8a>1%bHGK zbgD89FY`^J5(qpW+3*hTHO39*w6L8L^qpo+)$BS$CSnJ+`H7NDMbOLYX9_)pm3}z+ z^p+O^M5LxYNnb_|dZ4V`+D=+F3_6SS&vbmudrxc;F$zvE^03-pSp+R}Nz6Xn{n+xL`6WJ-%_S z6M4w&_*~N@7WUX<4u5~Ne;=WXNv4HKWb6!kuPb8k&8|clvZ~JXqFo&+s!YlNBPD{? zM+`DIlJ`|2Qn`wXKF)I)L>2ncBUf~zJ{+vXUMA3sqxZ!FE?(;kTonJj%wV_Swt9>7 zO|Ur3XN5T}EOo0DEydXF%|2HfaPzj<6M++;yG0vWdrzQy&iL6&60vSZ<$-&R3G~en z?U4Al>4vb#ahuiku*RMVd3ae24>nvK!JtP6zd{C3K5cOMBxZqD!o}y$3(WA#%qQ3E zazqTK`^_844GU5hYMUT?Y-R0lEz!hVno;YSbgc<20d{>YmD6g#xT>*T!ALBU&bU5J z#JSiViSVhG|3G>M7~Ue+TM+JI^kkNZvw3>@Sr=r}qr{+0PetRD8ivbRq^1<{FqR?V zZ(mnGU&0znbgBGj(nunLTpxax#QoGDQoF`u-JlbWOGKk(C8zgUOO~6NL!70xE{|AP zBOxvMcV}3M|3kr-zf_Ypr|T6~zlw;s}z4?lfRGhWdc|-}W_>kMqDsygdlwTTR zM66aXt7^`WdvTl=mm~i?L66hWcpO0LQdC;+YcBX!uMsGFE0$Nsv8ae#(dhafyFaJL zxjGKp&oELk>ff4e*7vcdyoMdg9VzuHNJ#@eMc6nIp>doWEu~`>i1cP%RJ=SG*5rIp z&%puZwNc9lNeM(>tfex=eKW&+Q>X9 zLk?*B8yazAgR$ew4$PIiFN+X7CcLfL!5w2Fy&KZBlk*#AH7DV zgmGkS^2FJV?6gcP9blR@^cNkNNQ8R-D&sSv+K^Hi(b%tc2Q9t(~+5nYp$lz~^Lg==w%Bz!jtWfcQuyY0wlZ3&zg3`hY8 zW*EdYM&mD{Muyb;W#{92+It7v^U`t@56=Hb>XiDwQaH)Q@5XM30isI&>f%(NQ{f*m z)aS6n(RsG%El}n<`}*!ygkmDzD!YsTxewI58v(Uv8E~X`9H;C2%RC;-RNVt0h3I+Z z^B-JQ#M;{0cz5oE6c)14(9&{O5!Z#-AggMGCD@D;$KI_#`cP9l={03lE%2p}#eToG zDlKraAa{pSM`13I&68NjPiGE{Pv;S8!K=GOj(BH{mBAZm=Y_v02v|B%O9 z7}Y;I?)(3<+6y5x;1L!kKi-}YK_0cQwt4Tf0*%QM&}npH83V7@lRThMXo^P$r9Qk@r((Q>K_#jXyXE$s`<5{` z60)_;Lk45MA}Auw4zytCp4iTf;0SC>K0ZFqo#%u)Q}OxOFk(81uRuv0-R;j1KX!0R zxmo+q51YlM$s_zdgkJX8O4gKcnx)sht_h2(Hy0gyi-|}WS4nO*rZvu_(S(*SZ|N%D z{GlQm{K3k_I_*~~rMBxU-}^kK4*`>Pqwtb*n^?m$fM9AkeFRd%^!89LE}fKLGUs-} z%{qIEf5>A7oCl}jW=Z$0yYtMl*u6gZO9C|O68<3lQX>*^&kbewNkG66-zDdg0c_GP zk}YciaGRn57Y%C==T|_HNEUTvIy>472fE3#!?n@BQ?(^`+QhuJUKtI{&(nivLP5N+ zswHQK)FR{976BzH>i##T_xV42w%&i96C|k{EZ2_ttRqN?g8_YOA(8U)7{zA+yf0qz zKJi<(AjmV1<1I6EZ^B)mTWe!Ra;PS9bpXaK5(qEZKtBIazJ0Vgoy>1@N9yXtN-ayu zOnrU0kQh+H(Kmu!hM0%WrNSF*1yXBTNguu2FLQuG{}OPq#XrDHEQ(uf|2Y>WccRF> zZu4BT(c)r=uUuzQ?A@D?qT7+a<*B4mWqZCERq;zpC#;?u0%sI~ zE-qgBt5ohXKESjUuvLuoqiNGu5!h;_OSNg5ZZ}&*n zHcp^t!}1QuNeL*A9b#X2I!@JSi!8xiW7Bwz4eSH{+$AQCuIAZG^JDgs0t*Lls(1dU3N2{?k_~r^@OKSdheAp@siR?c^S81V@^h zj5OXmMuvv9dD9gYcj)WIU7}lvkKZ>;=KRjMtsHtz+gK<}0lpld4d!O5&x)?}#LDS+ zz)(0G%64@1{^y>D4z;_y*Cugdj%FsqVvbFGDXb!H1@_T3j=i1tm6VhYAOAREl@@xb zexl<>4U2shM8RXKRr{pMANDDe*Fs7NCxLvz{+tDK2`E#s06`r#0e#6n1s2Cz^o9nO zU}n-%AD@`d^(gv#l)oaIY#Fu^7dNLs?IRB@}ID zy(L`P@xylTM7yLaM73h$lsvOt`UR2JwgbL6X@oJ5;SBe&Xw9?xusBg9oYnf?r1aH^ z93jTj0yRnibBXB$e^S7$G9`UI6HFweU_9B}N#tFnlHqP%nW;8ryI7ru+N_!{BJ6dA ze+07CJ|X?CG6rQl|3{dil7h85neH>!I-0oTv2?{?*j=6-vi@t1^6@{LFiJPS#;v1m z4k@qfi@~~{A1@@V`@F)Q5nj}eU3{pTWY^}%FGC8?<@VPTcAhKk;y~e~B-G4BQVH>2 z^v>5gJ|&8(2&%)Tf&~9)V6aZ!FdT zA}aR!k`E=YTEX1E4iQDj6H?*(fZuQsl||2dcK`e!D-B&8y{T|ILm7*&TVC8Zm@Oe_ z9iXKi!jRmXdsBY;+HyR8rKlycdXi($g!sNQ&In~P53yzwIJhz1d*~00oikIl4x`Bm z&%Xv?7O&2HdDf|?;*4)Eb$#I`q#iMzW}gQe2Pa%#mpg^c-vTW9EzqY*sE8I# z2w;>zv(xd=y`HqqDw2A#=Y`N|0(KC)qpSR`F4Ld-q47G?b&k5~WGu=EKt&&Q{p%jf zkBtfYT37kC(!1x5q@QKjev{5;qe+UFhs`y+eo}a)5pIy87M7F z#Ziu9(|nhemG%4Z$Qag? z5s2(p0ABPQ-*SxuB~!EyRyG72PWoS2un;P2=U_KQRMFqh==0wh+6}z;nD|>g-m*rQ=1rB$8K)qljOu)3%P^Rrz<%%W&s^UC zNo2A9SL#~nP`4Mk)^|xSMy~g~8UEb~#9BF5N~ws$3`ZGp28CK%8gaX*e8k4pS%gF< zaXhW#5_Uwa$@}Odh|HqGt&e=y%j(sR7~|v#9mY!eAPzAmlB>D}hjNGj3y@gsuWum7 z-^1_bbMM*{OMhg%lQ{)sTt^$T>}FLr@~PTquig1vItj}flSGg z%$5GMLVMC5;}(G#z1tDsli4%y53sJNF`M@tv}uG~lnn329yz_9^5JRWOlvc&_5cpB ze%JzY3U)1;vcUczZ#9RjkT7qm+E!D@>=rX6yFt|i%tDa+vI!XM+CJ=`RkUC6U!?sL z8bfZ{gCX}C9I4>x={WMzn*CCRGfG9Q2V*}pLC%AmF2U61qpUu&fG@WJ4%>}fts(Y>UIi#zmxT{_AI zL2Db`>pXz9VQl+6B473fY+8pt&&_plA0?m>TM0Y1qWF5dGH8I#vyDF6_VQCuNm9Po z5GQC(N>~?ZDlKljliGJ?C&C|{2nFVp%_$VG5J zbFe_4HjRv|wOmYw$eqzG?3N+gU#ncLVFuCv9wcos3R?1zOVvQdA8CP%ZU=Hg>MIK$ ziAKXaYc&Ezn{4V1cnYZhGist9wbDH+G+3-3J>QYvj#4@+ei<+x1wUgDg1+P8l{ zyhmj4LN2JrBR$T+3=_3~_PQ?VSE+9}qwK)1hQy8ANSuZ7LUwhvD}a9mqcZNAmAOn*2Ls z46#4BjlN(2E5o->a$zn@y$^uvZ=-hH8{I?lGerOwbO$dcm2UEZpxdf6jme>S-J1lg z7>aIfB7Kzyw^F09TJDC*wHEMzD=C=$htYrqD`;b~y2NtTWA)I-?`*a3?_@O_eEFwc z;^hr)Rq?Y;I}AEvu`b_lbIKS4}S4uqi>!EzV43g6bx z`8<*ha$oEs996rq10{=jaNZ&3!ZS299I3FNw;f35YaH<2UoP?n&6Q@Z*GqQ!3S{El&DH*LlW7a^?tKx-V2El(@v9uyZx*5tk#0^0Q3 zfPGP{wIBV%CE~SZ3S1P6!0@H``nBi{iUl4*JfOKp>8e8d2x!IbcZLxw0sWq9C+g;G zB48EWxvl#<$92_sL(Lr;7Id|)_dn0jzP7qw!|{;tVU) z;M?`d$$zBdWR&d#sf1sh_yxL)wd@awCrbxPfu7^O~22FvCW&W3!b)a)DVm$8Nab-74_OYJQ0#Wl^z2 zB3RyyXZasdg7)w&@In`Kh%?;mw{7Ax9*v0M!MAi@+_3Ef-aI*M!%i2d z6fMIBpVp(6%d)=IP9dbcXJN86Sm!wH0y@D55M-y`Cc-r+KK;;frh(*+(Iee9PLhYKG+e@K5GfS19H zTUGz!zk0UXFb^7>xP6C_&(9tT8r+QmFr=Vr=5TwS9^9Xk-ER6=AqN8jN`Co%UajmP zpYLp#@sN-h2=PsoC{n4w9Vq|WCFI~DQH&P^|NLgS$MBQXuh)Vx(E1-|fx3HqGC6{5 z!O`L2cT)xJp2OQRGBT({4HEC-;b{%~9BmX8s9GK@bYLa?Z~|`g9zgO@ZA*gB)nLN21zvZ9O8xZd7fGrJ0i9uQtkIq@GOI}Wd3JX@6weADs3 z{49jMycz!_0AqyyqeQtm*RVUN$Y32L+A1>>xv9fT4S^3TUuzO_c{94v-rv5$kIB8? z$Di@P3IRb!J?}TGZTY_gWm*-pEFSgKFj2$R5JY5KY}5#s5;d4TC`|?UFLd(eqQ)xj zMtDcur0KWJ4<|ZK+EdbFCITZn99J_8ogo!7} zp7_h%U0r|9x7(F~kzW67_09v%&%r9{(7>yfsk*SRu&DsR&F<{{maA@+zPUXrX4hgM zS65e`4J+s*BqjNPJ?lX!?5oGwfgO#QJD!Gy#tYx9n@F=ZC&i;ReoV@8LH;S;4EvUZ z3Zp~MZEQ!6j-`-{-@}ev&od=EqF9E1dpPsj|BaTqmK*0{pWpG1FS+2^pz&_ET}PP( zL9jyST74-1%keFiSxHYO?mnfS-mPL#1D^x(4u2WcQfl1-wT>I^ABXON|Ji{oXqR@E zk}Ps`3pKvf*BcJAu!E?jjG~h)z9qbS@nUWDJHv+>)L#y8PmNQV_rB1USEnFZA|8d(CJ3eOA`T0VmyzT z*noQxC2ARXWR*q*VUF=Pgpo&270?tk^Z%*0@)b{gA^mH4%RQo^KKGVBhE?9yrlTMn zzSBuO>p&qBtGN9H963rvKWGoAd_XVudL_e4tNMjHpg=m&aFi)8DC79I*zJ1RfZgvu zS_l{<0UF2C?|hJjZLmuQy)pMos%|C#qXU|x%3?3wq~`uqQDbBT6N?Q;UoUf|RaUHm zk`gm;BTAC>4N>i+TSHB#GvheCelNTjT~SUQ{ekiReQkGFvPr{zgFBVK7>Tz!sWtE2 z_5Hp^b8ju3c8HHJNP^ugJ#M+Buxb(mc7_exdjo_8mPS(}qhw^ZV%4PGeq@$KU1`xN zps~Iu^I54mNwc%#VPaykt|%xf7Jny(f6K>dlpP<-Fd|G7YVbSdmpb1N&}Tk5Sk>h+ zYq}4}a((|_H|=M-n8%$@pFY_K0CJYAD!R*t!+!!$E%`^{MgLJf{EZDbxVRGn!a-PKK8qwFkq$oP_fajw`|le=Qk!%|OCD7eWQ-*!8P_YluXMY*UOJ`C zD3_XtYdZ1>dml5340XOE)P<4n0fHA@bu-kl%+|KEdN1E0VgfS2a~og4UUGt-?X24VdF(MOSI<>kFPx4BI*CZx-A7cth*57?O(4c@Ibi8V&mWx z+kviLTjVoJ!U>RpoPUJM6^2qrknk?`^_tZ#>(umnfaQ^oqJfV5FP>*hq1kYAa;ApW z4@Am1OD!D1E~ite8VBotdzWHiWhD&I!)CX&5?`1mA~|%873pr4t`VZz__fN%L2XSDQ@l&bJ%m50eZH!p7m_GOcrdLS+0TL!3UujF2-5fD|R27jegciB(5 z``n#q;ysS&!>6R$UzCKx99+Km&IKAVR8{G=+Z*d3#8|9L~&-pLULhlr_bUp)6^jER| zqjWHd({WV+)FnVo=&D~ploQ>;!S=bnfZ@eR>oK6_py5NXRiZRg@AH-tlZ4uBe0$H00;p1hWNJtTK=q0p-RuIvup`CTJ4!}yR33A6s}l=4O;vZ*4@baP;mN6=x608I2`6hFHF zHAk<>3;!=fne6)VIO|SoUpz4fD@m|M9LCW4y`I0Gp4|#f~KAiQm3U9xpnNUIZE+>$PG* z-v!_}S&}Kp$a1qDKMOIbh>MGB1Cvo_r~4~urh{On;Q)B3N1GcPS6hJt9+6fLzmUR)#nf);WSLCDG7<0$K1lg1{Ng--2Q<2BOJaVK(5*V_&xjk_hsk7!h)rTDLC}h zwyp38NlD2?wFDvPmhO=UEI%Q^P(6IlS$vOA{F)sm)o`GuNP)@O$i*JaafY1-V zjGgLTgU|O)|LFt9;n!SMgzAw}$bxg*siGt($3$J2h%pNF+(HpVQXJo|5LUsinPK*KTbt9UmLeVQ~~K{c=Jr#$6fl8yQ>> zIP;sw9f?d$s8bbj$9fmQdmI9D4_k+Tq;~#{yT&9P6T!_qc&R4~8&(EWm)FVAsXKnA<^B*$g>+YsCQ-;sWYcLBjb@lJoOp!9G??$98 zg6H1wR!$|y$4E(n1NZ~Xpy+4umwB!M{m=+FbEYWl1`-h%un;O_5L?O)B<+)MLxpR# z6#h>HlaOYW8B&F9^3GyCY;25iJ#p_Ez<+&d2<&$G;WGqZ9qC>3f4>^~X~N5Q&vrhD zm6@-|U<_mn-Wh!rDW*P|9`!REu#!I;To?6Z7&~I}rW2PPNM``LTmV-J%=Z{b=94Sa z)TvOtg;S)D$aVnQX7TE8py9?{_~J)F^1Z-m(5AISC{~A+vU+y zAt}TwO;n4jfC8naDyg^fz8q*fd?dXtoh>^rvUb^L@aY#nrXD>^ajm_e_&eoGlA($U-ZkiKnB@9*0rF%Y6V9ReQFkD_mQ-%)^U2bjVq1)a@+o8)W zKY_w?=g+b{6giTaQ{FiS!$+Tua)T%9tk##36LU(lx#HZMGfYcwY$Qxf;mzS7Q2Nwxd|6Ijt~e6wY?QEyYSYCJDBCgcE>S<^(DmJ=l#BN z-Fy!@a{q97+@EyRJOqa7}V;yxH1e*o-ejf1_gg4q&9q^Fm zI|k_i%`sidp9&0*eFvr~7a;NqwOV$BoNQDNJ_fJ{*ev>PJ_~RR;G)=cpj*q%W|)|s zP7_i_qH6Debvcab%LjChR#P;Lmp~cSbH3wTH`FL&|CeX$YAOm?7 zmx^<8s<8Zwi3p8~iYls#j0mxT$;(q}yZo}+$PlLr!)m!#()U*K^Sffv5Bn_AiV0B` zIWc5uak^)5oR}z(WB7cJUYRy~5`w{geEv!#?AxL$%dfC*O}> zeq@-DH=G$=3CdEn)AQTbJX!APvyxd&oW#llg!Lx?C5shvz$|(?V1Ekck&=E>W1C~iL8T7dakH+kb zw%0TjpUq5FX5}r}S(VFR3cb{9*H<>TZ(~p7S5v?`Z}=gG8#e7ds(x`gsnhqEedK*> zewToK)TYt1ah%$x9f`)6&etOO@A!PUb_n;wBf-;-+tTNe9LZDT9fY; z%YKQYEaEit_?YwGWz)>n;v0H$DXvUBK=ms(ZpTn_Q;*5!PjMS<#sALhm8k@QFK<1Y zDbW;`5g7dgxv=qT{myTRkIZaz3np)QwQ4unzJ!k}gs|r-*J9YM3}l>u%J&;g*qKvs z8QlXG%_kr^={m&ST;_muNIO=v=?`wa*tI|^6BWfS9n*n^=x=TMvU36@p^8#2x!zwt z=@=h+4Gb1b6T$Gua5Hd2d_ezva zh|KTvbl>mK_xt`mKEL~)`;WMs*Lj|==Qy6naXb%@YqS1wkTbAvKPh7;{TTD2dGmS^8mN zoy%Yr{p88hdC;x!`h6kosvEU0nL!B&+VzCi%XrGbRK9?oN#$Kkaj|3OwguwP(OrL* zFua+CIq=#|d~`h9Lg$Iap9v7X4B@#IDCoQIWL)ok5nNF&KRG@27zH9J7IlXWAoe`lTp**) zCYPW@X=6Y;peLli6%-tt3Bd^XgTd$fvo{8ys?@*t0jSk>AQ8sBoowcRtj{|F@=sq7 zt&Z0Fh)5mp-1_ha(9$miJB4S`D-HShwjwMY8CF7zk#GX6X}IWN7x z$lKn0)4T(tV|vg*nee_Nl0SR=Wl;*z>t~Q(|oUfucZrlfR zl6(ZmFoX_ECtsUcE_x7HbwZLnJVXFKYtLWjHLC0?DZ+7ymCuB63B=y;WDu9C8-Mbp z0cw_jT@~Vf;I7)AsP+2V+^ZRC2$wNeWHI;jo#ItaiC%C=DJ|SSo+YNo!oniGFLM6$ zgdc02Eb80A_tTjd5-z_L55ExWeDcPo;))R>rsRJ5Z3RazS5#cQa29oioQVcsiI&D+ z1uJ{|XYXaKY@nqV5_W;N45if=Xsa&)c~TpD${na58gUq^SgiAJi}VU~J?A4u^x|%w zxfPi^{2b|(A(m*$_?@L)KgDTdsHaDX#kfEJqrrdnfga{B*T35Id1TyEm}vh%uI`zj zbp=8O7F8%$ypf@rjS)HPcD9P4VVv`8c}O5c0|4{Bf8*!t`<=(=gN^=>pmxetA=Nz1 zG}V&SeAD_`8{JEEbg7*i0T53Fk|jnG=lBJ(9;{p-CZHv`dMOg?t5N3t7Rz5dr6o!{ z?rpa}=g>j5W8sLWBtLvRkWFLQ*u;clu=7#&g=oQg$dmN_u@PU{LN8_;q$(z_sKm)E z*vxriC!czxU78X5nN|`vxok_G>V&9sMa`q;-S=vBXOs$8=cv(qPIu2i8A2|-D zxbMc}t`)!&L1Xd7Wo8w#A95YgM-8qG zD9dM@XETkvxdPSnujuhVF+RgTC_?4>zj?G=%g5+&hdwM>N@Htvi2}zn{${lgX86>t zY$XX!`$8TtuIZ}eD{sKQlA3*9G5LMcHu1+U;MD~Jn}imk>?hUh4(c|wEUc^=Cu!L@ ziGRl*-?WMO=hAdjh7r-SkTMK(A*<%K3e0wN@X#EyS%?AQ?!RstIUQw4xz_IHkYk&Q ziI%=?1&x~y`mWTE!U1f3mqp!~nUTs~*$>f#o7{sj25z?k((Na>)U>p;%HMrm-H`c0 zb^w=h?Or2B+%_c65Eg|nCSJSgk2Ij-)-zbP>`t~Q^GemjSEUf0w{#~L?6`Uj0dR{K z?y0TC2zG>PrR4J3h82lFxh@r%O_=y1N0~zIMhtJw|A#Mb{<}q9Y5v!^nf8w#<{zJO za$Q5elYU;TAk7x!-Tb`Tx@br+m>?|zdl zYh%N9n3DbX36R3>*B<$Ndn?N&6KkP60dJT7qL@k%lfk08LOF={$1_4%a0q||4`6o-jWI74ck6tza;FzjPf$;2PjH6V*Lw#FL@g7 z1nwnV)js~h^DCn{G+8>+=Ih|;RuU@fnh%AzwZACTwnG*6fXY_8uO%l>1F_F_b(D;l z_T%(h3+O%We9$ort^uam=b0I^W259<6?-l4aU#8RDFi6gxo8h8@6W*tB zMCZ8Y8C0TK{SkYcu5@WeplIA_igT9EJcHde5Yw6=PwAsgY+#3kf(_F557S5)ZJ`cR zN6}hWmQK?HTp_pnB}xiMd5!@*44O_$^^8&m=^}l_eW!OMo?IhO6h6iDd&|gT^7!*R zc`ZG=eq#O9bj}DCmhEp#>{Ie$EFW$@dGdtgCy-N2B!TBC+Siac{tijoNgPjm ztHJ>(_8(Rir?n*nNzW^)^Hnc#H&b(NYA0TdT8UJ66>L|jW$xk^%_bUyL#!J<@XknN zcLS=$0xz&YLE}9Fmhs{=50cRI389=mnZak7m~oqRLP=L5q~-tPXjzm?K$NAd z8^gLkTZM}zaAzP!aN^5{8@euo8FKhTB4?#BYY4PCe{!yq3e=F>Our$vGNJ68C07-d zmZ@Tp{Dz*nBnoy2F+<2ZwbaW>Kntw?dyfTwLN{>_CQ<)h!_bW~9|{tOi2$a0arF1U zvpb1&sxmW2;;viP=FkgfR-B6lSD+vu0kMnCfVmiu+J*Kkk-uN0INMxTU9rz??&nGr zwK}aESnn`AqHq%SVRn15t;DDG)2Mtr33fJ8hcaE^SBhfLpJ#!uO*ZymeElK!6PYE# zf!S~?u4`Ob4Ix$zUOF37jM0)-3~C!JWI1jDY<>|Cx`IsWy$ht2ZmK2nE|s#kBCNr# ze~QuY@k)2*9tQE}dn}SvIQ(-vb&}6tAZ|Ko2yR>{7R#ro zrO|hyeR+Tywdla#Xj0OtvG(}2+V!MRLB4=TA1}04`N?#Z>VYP^g)GZ(fL_n1pOl*{pM z$j9S#orM0`-VPs#aDV~>Wn6s?aoE^B0A%U4h~t}opUZ6@VZOGlKp%{H1~=2}HK`Rn zOb2_;EF}Z81mT!S0iS^Ek(K$638hhp_|4M<>dcF5$O~qWgUA zcffUP;&MXAPf0|s~*~G3P6d$eI;6de}7LeuOy#xcrWCO z*%6TDgyu!q$VosH9~OT}vx*;0wZ)%j8$K>EV_#F))#EW=OUlp!Gq~1$EsV#Y3{l&l zYfK;-6Kz-7Pvqjoup`hwLdDc6?PX+p=a z**fke0=EZj98HHP{dW44oILxywzlTmHvjxl1n~Bs_db)d&T9#WkN5J6K%E85AB~!W zG6Z#(&smF-=r5o6Q)9Apit4hjsm5_=46;fQyo9hBLz{9&_^X_K+a_A~%T}+x?$Td3 z{%Qh82^nFD4g-3x88bX3POAhC9lX|$@kk=O52^m)?u>Ni@;hm5Y!7{`;JfuCK_4^w zikf}ejY?ian;yawaYo6pIA?XizV#2WEm;6hiINv(i;1xl5B7`UxNrFn=+!nD$P5Vn zDVYZtrs{%h!(Y5Th}#v76d`;2_1RZc#PP)jg~&=q0eca$)}}MorJe5GU)SkO#Ry&% zci(g~d7V70-9R0~;Powc@1A)_a&v#3vV0{2QwLLNQ&W@EjNIwDT~iz-&Pf{E zgY!nTGR1GpYP z2&@r7z9dcit2iI$C*8fGWJY-3E`Gk^XI(AUf<$b+?SCS+YbU(iFxewIJpt~sI$zen z5-A16!!rm+2aj}rlaG&=AM-ZUiY}ZBE5^q3o#b0cNrxNQ4D6>2s)|_{&nWy81_L} z0luH`^7|QuNfWV4Qp`S1mn8My_3@i{RS)|zzM}Pb+S60Snm|gH&9{khlF{`-!-9>Ex{PfKRZ6Ok|*PbtAm5pU(InQO#BL zHY-r)srZaT0s;bXwn0mcwt0=b z01+nAKBT?3{X%_oEJ!oIMq(duwOj7tXlgV()dKi~heb7ANEv2lvVc~Tr){jQ*$X(S zz6Vd32wEZ4A?=P@Hj8}-!0LQHCr4f%Qh_OESP5wrI~0@dv29pu_{6gi)Mi@{~p`RquhI;OzX zm_=C@Y%KHbHnlO|D9uH2zsOxetH!@Q%oR!Vgz#2U7Zg*pW+~i<9Ca2W49EbCn?{Nao=Z9TC=h{h7RT(#DG(3`ds3}jAGL|wLd0B)6 z^r#3p_37lmR-bSit}_cv%r>{SWMN=-4H49tHk6mUl-ayZ_*asMMf0yD5#f~atH+IJ z=I7^&Vy3OS{EvR|)<;d4TtsO8hi5i$e7f_ZG|7plQdD(f>L$hN(=;hO2G8Ju9K4LK zJK{R8o1hFc42XwGoPIOQu`A+(2Bc3IMBlA*=S7IvQ0-(NyjIrbW}CmNB14rvxKAhj zWOCE+WNm%@CDg&3cQsSe``w@hvDg`BwZ6?Mq?Bw`syRxZdfD_KZ?m+aD7R)MJ;QgE zVCUeMi{Q#+x^-P>yE}f^E5l4gPbCSKL2R<^ZzRQ6`yX8sv3`~()E2IlD+z|9#OB-| zbjs|&mrVbi&e1{(q-NcZJ^O4WVd?F%u?FV~>SJH9%PT1rj!XR){BUdib5l3EH%729 z{N>O?4?En{io%e4=86}mZcqj>zWhR^L{gXc$iknLnvX$Yk*8aC?$;Q{n>bJ=QrP3} z#V>wL*L;Pc!umn!cl0dl6O7no5FZbg7u-`Q8?^q~?z?uQU_rOGXCLupMNo&yk+*N( zj&v^XiTz+BAtAZl*51UduZ5e^Rt6L+puF*@sd>}VWs?g`hP&c@X2`0Yx;R&vSK{=Y11jg(pH~tH9 zxV2u}Lo)J4J~Me;kj{?Xr2}U6p%G(Eo;jCpFSfj8;3d!sH-)FjFXSAi0$$&ie6YMKXhd zJ0z0eWWM=@3Mu_7CV2%MFMeNZ?d#K#B0-o1}@c+y6@<9dd{}sNFkhK<{lSNZy3%LtH z7A}p}xNrj3j?rowi0*@;H`D!3ZjdlaMGVG*nsDf+N`F8b=ln%|v|b0i)x3T=<`|;_ z7M+)J86`ZwU6amPfugDN5g)@?@GoLkC*EAXn{Ml)MaB3gcB00lm}ZCJA!RPIM~beo zQ<`7o&`ahA_=+qnYS(;*acEYQljFgj=(&2_&RChc&B|ygD@cp;o(Xir|D2clb40tR zvJ=!;G?zHDW5%=LITgIk2R8W?)3boM^=n##?xgg(>U-O$GX9^Ncz^O&T@}tRD8{Ri zP?pzS=1fWZW}5E08>CfLN;T#PvXDYquDc~~JD!LogW@%=>6U7Hz19 z=z`g2xP(N3Enz?2$HC}uI4DS}dAWfjqp>+HKSw+_S7Di+G@%WDqe1ABLD`~=-jBA? zhI?#bhrGd@Z$t_y`imeUv57u5UxRSt9RSW%*-0jl56K0^<4T_aaVHphimeZo{Gf*4 z|NEE5wD@l6P08m(%G{W-IQRRzw34^|!0-X8Jl#V zbVe`@=;0&Fja7#sI-3`2+M5M8*9kW>dO6A6!M^CskT}J~42596{^}_OzslyV@;#4` zFuForf3-?6m3Ah-m!m~5Lj=@;k=)c-eG?pM^}7cU;(ab4Pc(;n8+T|{e;6>(s=VBU znL4`<&`c(khaUq1p`Wy4s`$kU;P=qa}2ilpWa$C7$oIZEYk9wJp{v+P+{7Mr~XoPx|9Eg-iWp!D>ci#bEv=GbQsJ}P(Iz))sC8EGvW)eZNS12ztyA2HwM#7=|1BX| zRE5rjQth)%^wP#=e}vGuML}unUQ+9-2LmA;8{w53J;C+Zj`Shzm0Mg0^c50uuB}s= zbM8x^l_&L~z7u6mRa>92W7hq|Ezo_v%7MD|YzIIl(R$Y#p^z~#vc#Q8vOd^~BPX8i%k8Is_VQDTQD4JD1SFuy zKx!GVd)Jf6##KTHKp=1#Y#@Yk2ocge(lLDws7Qb3ZaF?#?SYzJq(rE;UbFC^LPZZ| z+vTnxtz!`{rjQWRlE5Thvrlwj$1L*Ew9+o=db zE+v1$-mN42J3IOI`9y80D2js%ha~#Q{5Z#fxF04LIz@N+n^zET1@Mo#guI7S4gUT| zfLH$0+S7yr%Od$koO6ftIZY=exitFB-NrsTdTsPqdDqRJp5IU@hhucWqJmu^^8Rd1 znVG3!h!MgS>9HijC0Q|Kl@TQ!#^zn03^!GjPHt6a;PdWFQPUEU=_gif##2d>2_fwdFm<8-MaU8e3=W$b5T-lPHWoIvPO+)v z$G;rq3&2cEVzuAvco$Aqxglb#C?}IhFtPg~dV-aNy(1IY}#i6Y8Cp#*z%+Mm=j2yIOtQi%@@h^~-N|X!-?b z$mvvOvRZ(0dYe}c-nA=xnj&nXR7+^f!l?(0p3Op z-xW}cl$2lhF$UKS^MMBsACx$@3a(1~P-lo;up1dtR66#i>-obG*09aHyx4BLWWvT$ zMqo)0{YJ_!3?X8gsx69H+fb+TlzVPH00?Y?4tb27q>#<^%bHqKM@Mh{~YO z7fi1t+dMybu6d#a7fG!*v+dB88Jt}hno$0JJF!K(+;1f$NPEM6e zc*L}nR^y;9(4&x`5v>H{l5+$E{AX`qe@JcZDOLL zOyH>lYf0K*j%|C^;O7wD(Oj1uti3u%;YOn4)QIaxFuL-9On*ad&~7Iexh*I;AZSpM zkzrCNo}RX@z}^TsXgBqo*dQVX0=YQ(tmD1CQ8{zb&P0%Y<9T;rIiH~h;WK=$S9}3t z^wLj{!#7j`)5_p*2m1b(1YV=ThA80rfsyZ9VAKTvMNEEM?#*?EZOhZ3 z6*2=SnL8JCynX1}hlqOuUuK<%V(O2;E}I`Esh9{E6%+QSt?r0eXxc0Q+~|RCLcTHu%3t(OT2N^%mmQ+lYu$ILPc2 zXPsUBZi$@3Otx%8M{Sj1B>G5rZeqZcMiEB5y0*9A{NIkLJlxN1h&i0~Y{|sTbo=Wp zZ$`$z@E$d`U^LieG>Twb@(IfQOb9{+=hg^U5%}zlFh?BRfkwYTW(fw5s*z`3pyk7L zd?$wo*m^faNo%%9yG!YTg?}K_nYBLK+TJt^6~sKAGIYXU5L<;sF@oni2S2+BpW`Tj zTf}GJbi|DFnSF{t=%8$5fM28Z(UjmM6#nhoH_@*hGL>%2gF0W^JvGvw99#jPCiUSE z6@RpZ*YA8OeNZ}&g2fTK;R3t`#)~xrb+^C&N=~Zw^<>3bpHeO9I-Yp)`gn?3wA7BP zQ}K=709FjmlLe_SFPb?sOt&?uMqS|d&>Zfd?UDuOh9Qhn(C!c{s(r^RxUZ@K`Qs+o z5X}Lkpe6NJeix`%{?b6%4uahH0RF+I2FP%;ks{MFP)KhzUy#l(@&PFRcGD0LhgA6c=SH#wqM*gB`<&UbDTh1>hE*?BNa z?05Zj+_A?OTVh^Ge_tGXSY5zP@T>Ff<5il;ULA^@I^Xc+=`r48VfYk7EH;`wWt;ot ztW9ABjUB0r-6<~+{tw^^!SUaLP9$e%#{)ZP!W$c9fLVr%Xc&6IoJj^23JpC*#2Ew} z<^dB>|DvrCtKSc%kgh%h96wY6aV`_cmX`#~L(oA`t4y4GNpjpzEyTdrEN9+ya^yFs zn_p{V5dHTviyrax%fJRVg%UX&--pFnYyxw!pHO9^lC5!61K=A~x-8QDDNa?JfMiH6UnRk#k3l`ic z1Bsn%f}(B`1c^hmrK(L4yj92IHE#UAjC{=odB2?wzvI|^lBQW${-ESMli*~3EJ-!$ zCgWB|h@$c_;bkPD9xS+v!T53?^A&Q^$8eyE>R(-jXmv|SbSq8LhYCJf9AWKb=R&~E zw<5TgO_PRm?eu8-0x}TLuUi1L0VbZTVq&yXQc}(*x1Z&yzT%n#1xzbFsfei@lNuEJ#|wXL;2C`t7Zq-3A`pRxesnuhh&}`5r3dG%(r>; zTXeZ<<12q8B%+Xia2ZYZYtf5V&~HE0%xi|p=sR*@9V+sUV>MfJr>Zky_lFO!F%R+e(@m$=G6J_)^=-4?Pg`vQ3YOtA{ zO(r5VRHwJ4tb$77s=>>I-#q&SstRdsSE9u7qb0Ahu(CLLACmtN+m{v`z@Hj3*}1lihT{jKyW?)iD5K(!~bnKHotXMXxl8#h? z88B=Fo397S5ut2;p+HlA0i+WaF!?~3Z~BdF-G^+>av-KN>=_%o(KMsFeZ&4!v_EHM zs<8Bx|48b32+yBa-IX#3jfoQ#6}?N(ivL3IGC8;0J%cj=Tt_^iw=d%xh8)uG1|;|% z_Q8iwRq|=wx~?_Uu8ZDObGw-l%R5`W#=IQn-u&e z0i^5c84#4cC?Cse-uuR6_f&;2R$KOf(4^KC%mb+e7~)o6WG^SkXz04xF4fJ0F_K6Z z_3w?Agn>oQ1+Zqk2@DzXK_&>Xnik`67(Ke2<%63l--+=4XBvEE{e925C*GG6)1?}x zjPSpmkFoJ4opdg`$GHD-7uVQ$(#iYhV{_sfj6AwX9`|F-sQod^Gw=pR#Bbibu{64a zZonpp=uaLT)q8I|2InJtBQE=Af|P>cw&=TpG%z(1x=(BH2QoZIBE^@l98}l-bvG)Z zRFPulEMwnmx?RF^X1HhmyY)-U>`|o`>TT4`k{*guF(12u5yj^o{FG}u((%JL3TE4z zrWyVyodlSu$2tSRSBQ_nF6j^tGvEi);(*X zu%}?}d?ls(m=wvwaJnW+)XmMaRcr4_Xv7M)dn(2L?Ge>!hH)S9b|{crYn?o~SB3q8 zmirPh=)Y0cm6(`eTcC$!x`n3(Jzfar#8J-TFOtZ0Z#Ii26fmGEd90UAIDr^_51