diff --git a/.dockerignore b/.dockerignore index bc16e556..95cf4c12 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store +compose*.yaml +bench-out +bin diff --git a/.env.example b/.env.example index 119384b5..700a36b4 100644 --- a/.env.example +++ b/.env.example @@ -6,12 +6,11 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Sessions are stored in Valkey on this stand. +SESSION_STORE_URL_FILE=/run/secrets/session_store_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2cc26e76 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Files read inside Linux containers or by sh must keep LF on Windows +# checkouts with core.autocrlf=true. +*.sh text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.cmd text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..4b763da0 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,60 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +defaults: + run: + # Explicit bash turns on `-eo pipefail`, so a failing check behind `| tee` fails the job. + shell: bash +env: + BASE_URL: http://localhost:8080 +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - run: go vet ./... + - run: go test -race -count=1 ./... + e2e: + runs-on: ubuntu-latest + needs: unit + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: make secrets + - run: chmod +x scripts/restart.sh scripts/stop-one.sh + - run: docker compose up -d --build --wait + - run: mkdir -p evidence-ci + - name: Image codecs in the worker runtime + run: make codec-test | tee evidence-ci/codec.txt + - name: Public smoke + run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt + - name: Persistence across a restart of the API and both stores + run: make test-persistence PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/restart.sh | tee evidence-ci/persistence.txt + - name: Failover of one replica behind the balancer + run: | + make test-failover PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/stop-one.sh | tee evidence-ci/failover.txt + docker compose start api + docker compose up -d --wait + - name: Concurrency and shared idempotency across replicas + run: go run ./tools/concurrency --base-url "$BASE_URL" --output evidence-ci/concurrency.json + - name: Container logs + if: always() + run: docker compose logs > evidence-ci/compose.log 2>&1 || true + - name: Cleanup + # The CI stand is disposable, so its volumes go too. + if: always() + run: docker compose down -v || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: evidence + path: evidence-ci diff --git a/.gitignore b/.gitignore index b5a71896..e1e41098 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +bench-out/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..02a6a856 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# Modules in their own layer and a persistent build cache keep the rebuild of +# a rolling update short. +FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download && go mod verify +COPY cmd ./cmd +COPY internal ./internal +COPY migrations ./migrations +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/worker ./cmd/migrate ./cmd/reconcile ./cmd/s3init + +# The load generator gets an image of its own; the API image does not carry it. +FROM build AS loadgen-build +COPY tools ./tools +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen + +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen +RUN adduser -D -H -u 10001 app +COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen +USER app +ENTRYPOINT ["/usr/local/bin/loadgen"] + +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS worker +RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp && adduser -D -H -u 10001 app +COPY --from=build /out/worker /usr/local/bin/worker +USER app +ENTRYPOINT ["/usr/local/bin/worker"] + +# Last, so that `build: .` builds it; it also runs the migrate service. +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 +RUN adduser -D -H -u 10001 app +COPY --from=build /out/server /out/migrate /out/reconcile /out/s3init /usr/local/bin/ +USER app +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab +EXPOSE 8080 +# The start interval lets a rolling update see a new replica healthy within +# about a second. +HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ + CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 +ENTRYPOINT ["/usr/local/bin/server"] diff --git a/Makefile b/Makefile index 5d86d171..096cb5a9 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,103 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --------------------------------------------------------------------------- +# Application targets (Go implementation). The targets above are the skeleton. +# --------------------------------------------------------------------------- +EVIDENCE_DIR ?= evidence + +.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout bench plot failover-load rollout-load + +build: + go build -o bin/server ./cmd/server + +vet: + go vet ./... + +# The tests start disposable PostgreSQL and Valkey containers: Docker must run. +unit: + go test -race -count=1 ./... + +secrets: + sh scripts/init-secrets.sh + +up: + docker compose up -d --build --wait + sh scripts/wait-ready.sh + +# Stops and removes the containers; volumes and therefore data are kept. +down: + docker compose down + +# Deletes the volumes too: every user, chat, message and session of the stand. +destroy: + docker compose down -v + +logs: + docker compose logs + +# Safe default: inspection only; APPLY=1 explicitly enables repairs. +APPLY ?= 0 +DEEP ?= 0 +.PHONY: reconcile upload-memory reconcile-bench +reconcile: + docker compose --profile recon run --rm reconcile $(if $(filter 1,$(APPLY)),--apply,) $(if $(filter 1,$(DEEP)),--deep,) + +upload-memory: + $(PYTHON) -X utf8 scripts/lab4/upload-memory.py --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/upload-memory.json" + +reconcile-bench: + go run ./tools/reconcilebench --output "$(EVIDENCE_DIR)/track2/results.json" + +# Which container answers under which X-Instance-Id. +instances: + sh scripts/instances.sh + +# Replaces every API replica without downtime; all of them must be running. +rollout: + sh scripts/rollout.sh + +# Track 1. Every run starts from empty volumes, so the stand's data is lost. +# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200 +SERIES ?= scripts/bench/series/A.conf +bench: + sh scripts/bench/run-series.sh $(SERIES) + +# Needs matplotlib from requirements-bench.txt: make plot PYTHON=. +plot: + $(PYTHON) scripts/bench/plot.py evidence/track1/raw evidence/track1 + +# Level 4 under load: one replica stopped (KIND=stop) or killed (KIND=kill), +# and the rolling update. +KIND ?= stop +REPLICA ?= 1 +failover-load: + sh scripts/bench/failover.sh $(KIND) $(REPLICA) + +rollout-load: + sh scripts/bench/rollout-under-load.sh + +concurrency: + go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" + +explain: + docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql + +# Set CODEC_STRESS=1 after generating bin/decoder-fixtures to include the 40 MP cases. +.PHONY: codec-test +codec-test: export GOOS = linux +codec-test: export GOARCH = $(shell docker version --format "{{.Server.Arch}}") +codec-test: export CGO_ENABLED = 0 +codec-test: export MSYS_NO_PATHCONV = 1 +codec-test: + go test -c -o bin/imaging.test ./internal/imaging + docker build --target worker -t mtusi-messenger-worker:lab5 . + docker run --rm --network none --cpus 2 --memory 2g --entrypoint /codec/imaging.test -e CODEC_REQUIRED=1 $(if $(filter 1,$(CODEC_STRESS)),-e CODEC_FIXTURES=/codec/decoder-fixtures,) --mount "type=bind,source=$(CURDIR)/bin,target=/codec,readonly" mtusi-messenger-worker:lab5 -test.v + +# Explicit destructive-to-availability experiment on this lab's own stand. +# Containers restart; volumes and accepted attachments are preserved. +.PHONY: track2 +track2: + docker compose -f compose.yaml -f compose.faults.yaml up -d --build --wait + $(PYTHON) -X utf8 scripts/lab5/track2.py --project "$(or $(COMPOSE_PROJECT_NAME),lab5)" --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/track2/run.json" diff --git a/REPORT.md b/REPORT.md index e98f75e9..5df91310 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,145 @@ -# Отчёт — лабораторная 5 +# ЛР 5. Асинхронная обработка вложений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка — **5**, трек 2. Проверяем гипотезу, что после успешного HTTP-приёма рестарты и повторные доставки сохраняют задание, а устаревшая попытка не может заменить опубликованную версию. -## Запуск из чистого clone +Автор — Бакулин, БПИ2301. База переноса — ЛР 4, коммит `80c4a56200e3`. Baseline для сравнения изменений ЛР 5 — `911d97a`. Проверенный код и сценарии эксперимента зафиксированы в `b05b338`. Протоколы с измерениями приложены в каталоге `evidence/`. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Запуск из чистого клона -## Архитектура и решения +Понадобятся Docker Desktop с Compose v2, Git Bash, Make и Python 3.9+. Для локальных Go-тестов нужны Go 1.27.1, C-компилятор для race detector и работающий Docker. Команды ниже для Git Bash. Переменные портов позволяют одновременно держать стенд четвёртой работы. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```sh +export PYTHONUTF8=1 +export PUBLISH=127.0.0.1:18085 STATS_PUBLISH=127.0.0.1:18405 BASE_URL=http://localhost:18085 +cp .env.example .env +make secrets +make up +make check test PYTHON=python +make unit GOFLAGS=-p=2 +make vet +``` -## Доказательства критериев +`make secrets` создаёт отдельные секреты PostgreSQL, Valkey, RabbitMQ и S3. Храните их вместе с томами стенда. Compose запускает миграции и инициализацию приватного бакета перед API и worker. `make down` останавливает проект с сохранением данных. Команда `make destroy` удаляет тома проекта, включая историю, задания, очередь и изображения. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Как проходит запрос -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`GET /health/ready` проверяет, что API ещё принимает трафик и может обратиться к PostgreSQL и Valkey. Эти зависимости нужны для чатов и сессий. При отказе S3 текстовые запросы продолжают работать, а операция с файлом может вернуть 503. При остановке RabbitMQ или worker API по-прежнему может надёжно принять загрузку, пока доступны staging и PostgreSQL. Health-check самого worker в эту проверку API не входит. -## Эксперименты и отказы +API проверяет членство в чате, потоково сохраняет исходник в SeaweedFS под ключом `staging/`, считает SHA-256 и проверяет изображение. Перед полным декодированием применяются лимиты 10 MiB и 40 млн пикселей. На каждой API-реплике одновременно работает один декодер. Как в ЛР 4, upload требует положительный Content-Length. В Postman режим binary отправляет длину файла. Потоковая загрузка chunked без объявленной длины получает 400. Одна транзакция создаёт строку общего файла при её отсутствии, вложение и задание первой версии. После COMMIT API возвращает снимок `202 queued`. Клиент уже может прикрепить вложение к сообщению. Скачивание до первой публикации даёт 409. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Строка `attachment_jobs` одновременно хранит попытку обработки и намерение отправки в RabbitMQ. Каждая API-реплика выбирает одну созревшую строку с `FOR NO KEY UPDATE SKIP LOCKED`, отправляет persistent-сообщение с обязательной маршрутизацией и ждёт publisher confirm. Только затем помечает попытку отправленной. При падении до фиксации та же логическая работа может попасть в очередь повторно. Тайм-аут штатного ожидания confirm — 5 с. В сообщение входят служебные поля схемы jobs, включая стабильный `job_id`. Worker берёт ключ источника, размер и SHA-256 из PostgreSQL. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +При отправке сообщения транзакция фиксирует целевую версию вложения через `FOR SHARE` на `attachments` и проверяет статус под `FOR KEY SHARE`. Эта блокировка совместима с блокировкой публикатора, поэтому очередь может ждать подтверждения одновременно с записью сообщения. В тесте с открытой транзакцией публикатора сообщение получило 201 за 147 мс. Переход в `failed` берёт `FOR UPDATE`. Когда первая обработка завершается ошибкой до отправки сообщения, отправитель получает 409, а строки сообщения, ключа идемпотентности и связи откатываются. -## Ограничения и следующие шаги +Worker проверяет актуальность сообщения и получает ограниченное по времени право обработки — lease с уникальным токеном. После чтения он сверяет размер и хэш оригинала. Результаты хранятся в SeaweedFS. Общий оригинал имеет ключ по хэшу, ключ варианта содержит ID вложения, версию, имя и хэш выходных байтов. Одна транзакция проверяет целевую версию и токен, фиксирует сразу три варианта и переключает опубликованную версию. Только после устойчивого эффекта consumer отправляет ack. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +При повторной обработке пользователь продолжает читать прежнюю публикацию вплоть до переключения. Внутреннее состояние операции v2 и публичная доступность v1 различаются. Одна версия алгоритма обрабатывает одно логическое вложение. Два чата с одинаковой картинкой разделяют физический оригинал, у каждого вложения свои права доступа, задания и версии. + +Миграция 00004 сохраняет UUID вложений ЛР 4 и ссылки сообщений. Для прежних готовых оригиналов создаёт задания v1 с источником в blobs. Утраченный ранее оригинал остаётся доступен для диагностики с кодом source_missing. Таблица attachment_variants нужна для атомарной публикации MIME, размеров и SHA-256 каждого из трёх результатов. Таблица attachment_replays хранит, кто явно запустил новую попытку и после какой ошибки. Без этой записи операторское действие было бы неотличимо от автоматического retry. Реестр неудач читается непосредственно из attachment_jobs. + +## Изображения и лимиты worker + +Оба профиля создают JPEG-миниатюру до 256×256 без увеличения и центральный JPEG-crop 128×128. У JPEG качество 85, прозрачность компонуется на белом. Для WebP изображение вписывается в 1280×1280 с сохранением пропорций. У v1 качество 80, у v2 — 64. Применяется EXIF orientation, выходные файлы сохраняются без исходных метаданных. Оригинал выдаётся без изменений. + +У PNG исходный холст может быть больше растра и иметь ненулевое смещение. Сразу после чтения worker сбрасывает координаты холста. Центральный crop вычисляется по растру с учётом ориентации. Для сплошного красного растра 320×200 с холстом `2400×1800+850+30` проверили скачанный через API JPEG. На высоте 64 в точках x = 1, 64 и 126 получили RGB (254, 0, 0). До исправления центр был белым. + +Worker запускает кодек дочерним процессом с общим дедлайном 45 с или более ранним дедлайном задания. У ImageMagick ограничены потоки, память и дисковый кэш. Для допустимого PNG шириной 40 млн пикселей рабочая строка обычного декодера превышала лимит контейнера. Такой PNG обрабатывает отдельный Go-процесс, который уменьшает изображение и сохраняет центральный фрагмент перед кодированием. На время этой работы резервируются оба слота. На трёх реальных файлах по 40 млн пикселей получены все варианты в контейнере с лимитом 2 GiB. [Методика, размеры и измерения](evidence/codec-40mp.md) + +Рабочие файлы кодека хранятся в каталоге попытки внутри `/tmp/messenger-worker-imaging`. При обычном выходе worker удаляет каталог попытки. При старте процесса, до запуска потребителей, он очищает свой каталог от прерванных попыток. В эксперименте с JPEG CMYK на 40 млн пикселей `SIGKILL` оставил 82 680 KiB рабочих данных. После запуска того же контейнера старый каталог исчез. При штатном бюджете 60 с то же вложение перешло в `ready` на второй попытке. Клиент увидел три варианта, рабочий каталог был пуст. Исходник для повтора хранится в S3. + +## Повторы и эксплуатация + +Значения по умолчанию — 2 потребителя с prefetch 1 каждый, бюджет обработки 60 с, lease на 5 с дольше, до 5 попыток. Для временной ошибки рассчитывается задержка с экспоненциальным ростом, верхней границей в минуту и jitter. Отказ БД до выдачи lease может вызвать повторную доставку без запуска обработки. По этой причине транспортный лимит доставок отключён, а бюджет реальных попыток хранится в SQL. У RabbitMQ 4.1 quorum-очередь по умолчанию отбрасывает многократно возвращённые сообщения. [Документация RabbitMQ](https://www.rabbitmq.com/docs/4.1/quorum-queues#poison-message-handling) + +Истечение lease обнаруживают API-реплики. Исчерпавшие бюджет и постоянно ошибочные задания попадают в реестр failed. Повтор запускает оператор. CLI проверяет доступность исходника, записывает оператора и предыдущую ошибку, выдаёт новый бюджет с монотонным номером попытки. Запрос replay завершённого задания получает отказ. + +Для `worker failed` достаточно соединения с PostgreSQL. Командам `replay` и `reprocess` также нужен S3. В журнале ошибки обработки worker выводит этап и причину, например `stage=read_original reason=source_missing`. Публичный `error_code` позволяет клиенту узнать исход задания. В запись об ошибке попытки не попадают сырые тексты ошибок хранилища, URL и ключи доступа. + +```sh +docker compose exec worker worker failed +docker compose exec worker worker replay +docker compose exec worker worker reprocess 2 +``` + +Сборщик мусора защищает staging всех актуальных queued/processing-заданий независимо от возраста. Для failed исходник удерживается после последнего перехода состояния (по умолчанию сутки). Для сирот применяется защитный интервал (по умолчанию час). Отдельные ключи доступа разрешают обход бакета сборщику. Оператор может проверить находки командой `make reconcile`, для применения исправлений используется `make reconcile APPLY=1`. + +Сборщик сопоставляет ключи каждой страницы S3 с заданиями и опубликованными вариантами одним SQL-запросом. Перед удалением он повторно проверяет кандидата под блокировкой и сверяет возраст объекта. В тесте с PostgreSQL и тестовой реализацией интерфейса S3 на 300 опубликованных вариантах, 100 активных staging-объектах и одном оригинале число SQL-запросов уменьшилось с 403 до 5. В обоих случаях было 7 вызовов S3 и 0 удалений. + +## Показ через Postman + +1. Создать пользователя, получить сессию, создать чат. Для дальнейших запросов включить Bearer-сессию и использовать префикс `/api/v1`. +2. Выполнить `docker compose stop worker`. В Postman загрузить `tests/fixtures/sample.png` через `POST /chats//attachments?filename=sample.png`. Тело — binary, заголовок `Content-Type: image/png`. Сохранить ID из ответа 202. +3. Отправить `POST /chats//messages` с новым `Idempotency-Key` и телом `{"text":"","attachment_ids":[""]}`. Получить 201. Запросить `GET /attachments//content` и увидеть 409. +4. Перезапустить API и брокер с сохранением томов. Запустить `docker compose start worker`. Опросить `GET /attachments/` до ready. Для исходника 320×200 получить thumbnail 256×160, crop 128×128 и optimized 320×200. Скачивание каждого результата идёт через тот же API с `?variant=thumbnail`, `crop` или `optimized`. +5. Командой reprocess заказать версию 2. Проверить SHA-256 опубликованного WebP. Для точных остановок перед COMMIT, после записи объектов и перед ack используется автоматизированный эксперимент. + +19 ноября полный сценарий защиты воспроизведён на отдельном Compose-проекте. Worker был выключен ещё до upload. До его запуска два сообщения были видны после перезапуска обеих API-реплик и брокера, а скачивание до рестарта вернуло 409. После запуска тот же job завершился на первой попытке. Скачанный оригинал совпал с отправленным файлом, JPEG имел размер 256×160. [Порядок действий, ID и контрольные суммы](evidence/worker-off-restart.json) После исправлений тот же сценарий прошёл повторно. [Протокол на последней сборке](evidence/worker-off-after-fixes.json) + +При остановке после записи S3 и до фиксации результата следующий исполнитель имеет право повторно записать те же байты. Видимый эффект остаётся один, поскольку одинаковые байты получают тот же ключ, а транзакция допускает только одну публикацию. Если новый профиль уже получил право публикации, запоздавший исполнитель старого профиля не пройдёт проверку версии и lease. Поэтому гарантия формулируется как идемпотентный эффект при доставке at least once. Число доставок и число фактических вызовов кодека может превышать единицу. + +### Что показали остановки в конкретных точках + +| Точка остановки или ошибка | Что удалось проверить снаружи | +| --- | --- | +| Исходник записан в staging, SQL-транзакция ещё не завершена | После принудительной остановки API строк вложения — 0. Уборщик нашёл сироту после grace, удалил его, повторный обход подтвердил удаление. | +| Задание зафиксировано, RabbitMQ недоступен | API принял вложение и сообщение. Обе API-реплики перезапущены. После возвращения брокера тот же ID перешёл в ready. | +| RabbitMQ принял сообщение, SQL ещё не пометил попытку отправленной | После остановки публикатора получены две копии в очереди. При выключенном worker обе остались после рестарта брокера с томом. В итоге опубликованы три варианта одного задания. | +| Worker остановлен при получении lease или после записи объектов | Попытка 2 завершила то же задание. До COMMIT SQL показывал 0 вариантов, даже когда в S3 уже были все три объекта с ожидаемыми хэшами. | +| Worker остановлен после COMMIT и до ack | Все три варианта уже видимы. После повторной доставки прежний job_id остался на попытке 1. | +| Первый исполнитель одной версии задержан дольше lease | Второй исполнитель опубликовал результат. После возобновления первого остались хэши победившей попытки. | +| Пока v1 удерживается перед COMMIT, завершилась v2; затем доставлены старая попытка и дубли | Для проверочной картинки SHA-256 WebP v1 начинается с `544068e1`, v2 — с `b99d07b6`. После последних подтверждений HTTP вернул WebP v2, в SQL опубликована версия 2 с тремя вариантами. | +| Недоступный источник и отдельный случай подмены источника | Временная ошибка закончилась failed на пятой попытке. После устранения причины replay завершил шестую попытку с одной записью аудита. Несовпадение контрольной суммы привело к failed на первой попытке. | + +Первый протокол включает отдельный повтор сценария потери подтверждения публикации с рестартом самого брокера. Он сохраняет идентификаторы, состояния и хэши HTTP-ответов для каждой точки отказа. [Первый прогон](evidence/track2/run-1.json) + +Во втором прогоне все 11 сценариев выполнены подряд за 572,230 с. Для каждого созданы новые синтетические вложения в том же Compose-проекте с сохранёнными томами. В обоих прогонах совпали контрольные суммы для обеих версий, порядок подтверждений 2 → 1 и количество опубликованных вариантов. В первом прогоне основная матрица заняла 629,758 с, дополнительный замер перезапуска брокера — 159,970 с. Эти времена включают остановку и старт контейнеров, ожидание lease и проверку результатов. [Второй прогон с временами по сценариям](evidence/track2/run-2.json) + +## Проверки на стенде + +После исправлений тимлидского ревью полная матрица прошла 11 из 11 сценариев за 479,143 с. В оболочке задавались один потребитель и одна попытка. Экспериментальный Compose-профиль воспроизвёл двух потребителей и пять попыток. Сообщение с вложением сохранилось ещё при открытой транзакции публикатора. При рестарте брокера обе копии задания сохранились. После подтверждений в порядке 2 → 1 осталась опубликована v2. После исчерпания пяти попыток операторский replay завершился на шестой, с одной записью аудита. [Протокол после исправлений](evidence/track2/run-after-fixes.json) + +На этом же стенде команда `worker failed` вернула одну запись реестра с заведомо недоступными настройками RabbitMQ и S3. Журнал обработки для исчезнувшего и подменённого исходников показал `stage=read_original` с причинами `source_missing` и `source_mismatch`. + +`make upload-memory` измеряет приём от отправки запроса до ответа `202 queued`. Скрипт учитывает только ответы с корректными метаданными, затем дожидается `ready` для одного контрольного вложения каждой серии, скачивает оригинал и сверяет байты и MIME. В обычном профиле обе серии достигли десяти одновременных запросов. Все 20 загрузок получили разные ID. Для PNG по 1 MiB p95 составил 352 мс, для PNG по 10 MiB — 3103 мс. Оба скачанных файла совпали с отправленными. У API-реплик не было OOM и перезапусков. Память снималась периодически через `docker stats`, поэтому кратковременные пики могли пройти между отсчётами. [Измерения загрузки и памяти](evidence/upload-memory.json) + +19 ноября после правок по ревью все 11 сценариев повторены за 399,954 с в отдельном проекте `lab5-review`. Обе копии пережили рестарт RabbitMQ. После возобновления v1 и дополнительных доставок опубликованной осталась v2 с тремя вариантами и прежним SHA-256 WebP. Replay завершился на попытке 6 с одной записью аудита. [Протокол прогона после ревью](evidence/track2/run-review.json) + +| Свойство | Проверка | Наблюдение | +| --- | --- | --- | +| Пакеты Go и конкурентный доступ | `go build ./...`, `go vet ./...`, `go test -race -p 2 -count=1 ./...` | Прогон с временными контейнерами завершён с кодом 0. [Вывод race detector](evidence/go-race.txt) | +| Целостность контракта | `make check`, `make validate` | Локальные ссылки, публичные Python-тесты, полный OpenAPI 3.1 и JSON Schema прошли проверку. | +| Приём без обработки в HTTP | `make test`, эксперимент с выключенным worker | Получен 202, ID сразу прикрепляется к сообщению. Оригинал и JPEG доступны после ready. | +| Сохранность подтверждённых данных | `make test-persistence` с restart-hook | После остановки и старта проекта сохранились прежняя сессия, сообщение и вложение. | +| Доступность чата при остановке одной API-реплики | `make test-failover` с stop-one-hook | До остановки наблюдались две реплики. За окно публичного теста прошли 119 записей, ошибок 0. Первая успешная запись после завершения hook зарегистрирована через 8,203 с от его запуска. | +| Устойчивость доставки | `go test -race ./internal/broker -count=1` | 22 разрыва соединения до ack пережиты одной подтверждённой публикацией. При заводском лимите quorum-очереди контрольный тест терял её на 22-й доставке. | +| Лимиты преобразований и корректность форматов | `make codec-test`, отдельный прогон с `CODEC_STRESS=1` | JPEG и WebP декодируются, выдержаны форма, поворот, фон, размеры и параметры двух профилей. Прогон трёх предельных исходников занял 43,10 с. | + +19 ноября на сборке после ревью в обычном режиме Compose повторно прошли smoke, persistence и failover. В новом окне failover выполнено 118 записей, ошибок 0. Первая успешная запись после завершения hook зарегистрирована через 6,369 с от его запуска. В обычном профиле после тимлидского ревью вновь прошли все три проверки. Последнее окно failover дало 131 успешную запись, 0 ошибок, первую запись после hook на отметке 6,168 с от его запуска. [Команды и результаты прогонов](evidence/public-checks.txt) + +Для последних двух публичных тестов на Windows в Git Bash используются команды ниже. Для Linux заменить `.cmd` на `.sh` и дать скриптам право исполнения. + +```sh +make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.cmd +make test-failover PYTHON=python ACTION_SCRIPT=scripts/stop-one.cmd +docker compose start api +``` + +## Пределы эксперимента + +Docker Desktop 28.5.1, 8 CPU, 12 547 280 896 байт памяти в Docker VM. Две API-реплики ограничены 1 CPU и 2 GiB каждая, worker — 2 CPU и 2 GiB, брокер — 1 CPU и 512 MiB. На одном хосте проверяются отказы процессов и рестарты с сохранением томов. Потерю единственной копии данных при разрушении хоста эксперимент не покрывает. Медленный worker при работающем API увеличивает очередь и объём staging. Исследование длительной перегрузки относится к другому треку. + +## Проверка трека 2 + +Для отдельной копии стенда задайте своё имя `COMPOSE_PROJECT_NAME` и свободные порты `PUBLISH`, `STATS_PUBLISH`, `BASE_URL` до запуска `make track2`. Экспериментальный профиль задаёт двух потребителей и пять попыток независимо от настроек оболочки. Он сокращает timeout обработки до 15 с, поэтому lease истекает через 20 с. В обычной конфигурации эти интервалы составляют 60 и 65 с. + +На отдельном стенде с экспортированными портами выполнить `make track2 PYTHON=python`. Команда запускает Compose с экспериментальным профилем и выполняет отказные сценарии. В этом режиме API и worker могут ждать в указанной точке обработки конкретного синтетического задания. Сценарий останавливает и запускает компоненты выбранного Compose-проекта. Обычный запуск не включает эти точки остановки. После эксперимента `make up` вернёт обычную конфигурацию. По умолчанию результат записывается в `evidence/track2/run.json`, каталог меняется через `EVIDENCE_DIR`. + +В матрице проверяются остановки до и после фиксации задания, после подтверждения RabbitMQ, после получения lease, после записи объектов и после фиксации результата до ack. Порядок событий задаётся явными барьерами, а не задержкой предполагаемой длительности. Клиент сверяет состояние, число вариантов, хэш скачанного оригинала и байты каждого опубликованного варианта с PostgreSQL. При сравнении версий используется детерминированная текстурная картинка, которая даёт разные WebP при разном качестве сжатия. + +## Источники + +Контракт лабораторной, собственная реализация ЛР 4, RabbitMQ server и Go-клиент amqp091-go, SeaweedFS, PostgreSQL и кодеки изображений. Перечень зависимостей приведён в go.mod и Dockerfile. + +Формулировка для публикации после ручного воспроизведения автором лабораторной работы: + +ИИ-ассистент использовался как парный программист: генерация кода, тестов и черновиков документации по моим архитектурным решениям с последующим ревью каждого модуля и переработкой по замечаниям; выбор стека, структура проекта, модель данных, обработка ошибок, логирование и границы модулей определены мной; все проверки воспроизведены мной вручную. diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..851b2e85 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,49 @@ +// Command migrate applies the embedded schema migrations once and exits, so +// the API replicas never race each other over the schema. +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" +) + +// poolSize covers the goose session lock and the migration itself. +const poolSize = 2 + +func main() { + cfg, err := config.LoadMigrate(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + log := cfg.NewLogger() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg); err != nil { + log.Error("migration failed", "error", err.Error()) + os.Exit(1) + } +} + +func run(ctx context.Context, log *slog.Logger, cfg config.Migrate) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, poolSize) + if err != nil { + return err + } + defer pool.Close() + applied, err := postgres.Migrate(ctx, pool) + if err != nil { + return err + } + log.Info("migrations applied", "applied_migrations", applied) + return nil +} diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 00000000..03c63250 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,100 @@ +// Command reconcile reports attachment inconsistencies as JSON lines. Mutations +// require --apply; --deep also streams referenced objects to verify SHA256. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "messenger/internal/attachment" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, out, errOut io.Writer) int { + flags := flag.NewFlagSet("reconcile", flag.ContinueOnError) + flags.SetOutput(errOut) + apply := flags.Bool("apply", false, "apply repairs (default: read-only dry run)") + deep := flags.Bool("deep", false, "stream referenced objects and verify SHA256 and size") + grace := flags.Duration("grace", time.Hour, "minimum file/object age; must be at least 2m") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() != 0 || *grace < 2*time.Minute { + fmt.Fprintln(errOut, "reconcile: no positional arguments allowed; --grace must be at least 2m") + return 2 + } + databaseURL, err := config.LoadDatabaseURL(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + // Compose mounts the reconciler identity at the standard credential paths. + s3Config, err := config.LoadS3(os.LookupEnv) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + blobs, err := s3store.New(s3Config) + if err != nil { + fmt.Fprintln(errOut, "configuration error:", err) + return 2 + } + db, err := postgres.NewPool(ctx, databaseURL, 10) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 1 + } + defer db.Close() + r, err := attachment.NewReconcilerWithRetention(db, blobs, *grace, s3Config.StagingRetention) + if err != nil { + fmt.Fprintln(errOut, "reconcile:", err) + return 2 + } + // A broken output stream cancels the scan so repairs cannot continue after + // the caller loses the audit trail. Print partial statistics on scan errors. + scanCtx, cancel := context.WithCancel(ctx) + defer cancel() + encoder := json.NewEncoder(out) + var outputErr error + options := attachment.ReconcileOptions{Apply: *apply, Deep: *deep} + stats, runErr := r.Run(scanCtx, options, func(f attachment.Finding) { + if outputErr == nil { + outputErr = encoder.Encode(f) + if outputErr != nil { + cancel() + } + } + }) + if outputErr == nil { + outputErr = encoder.Encode(stats) + } + if outputErr != nil { + fmt.Fprintln(errOut, "reconcile output:", outputErr) + return 1 + } + if runErr != nil { + fmt.Fprintln(errOut, "reconcile:", runErr) + return 1 + } + return 0 +} diff --git a/cmd/reconcile/main_test.go b/cmd/reconcile/main_test.go new file mode 100644 index 00000000..806f2d67 --- /dev/null +++ b/cmd/reconcile/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReconcileDoesNotValidateUnusedLogging(t *testing.T) { + for key, value := range map[string]string{ + "DATABASE_URL_FILE": "postgres://test:test@127.0.0.1:5432/messenger?sslmode=disable", + "S3_ACCESS_KEY_ID_FILE": "test-access", + "S3_SECRET_ACCESS_KEY_FILE": "test-secret", + } { + path := filepath.Join(t.TempDir(), key) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + t.Setenv(key, path) + } + t.Setenv("S3_ENDPOINT", "http://127.0.0.1:8333") + t.Setenv("S3_REGION", "us-east-1") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("APP_LOG_LEVEL", "not-a-log-level") + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Stop before dialing: only configuration relevant to this CLI is checked. + var out, errOut bytes.Buffer + code := run(ctx, nil, &out, &errOut) + if code != 1 || strings.Contains(errOut.String(), "APP_LOG_LEVEL") { + t.Fatalf("unused logging prevented the database operation: code=%d error=%s", code, errOut.String()) + } +} diff --git a/cmd/s3init/main.go b/cmd/s3init/main.go new file mode 100644 index 00000000..164c3b01 --- /dev/null +++ b/cmd/s3init/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "bucket bootstrap failed:", err) + os.Exit(1) + } +} +func run() error { + cfg, err := config.LoadS3(os.LookupEnv) + if err != nil { + return err + } + s, err := s3store.New(cfg) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return s.Bootstrap(ctx) +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 00000000..90904146 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,200 @@ +// Command server starts the messenger API over PostgreSQL, Valkey and S3. +package main + +import ( + "context" + "expvar" + "fmt" + "log/slog" + "net" + "os" + "os/signal" + "runtime" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/app" + "messenger/internal/attachment" + "messenger/internal/chat" + "messenger/internal/pipeline" + "messenger/internal/platform/config" + "messenger/internal/platform/ident" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" + "messenger/internal/platform/valkey" + "messenger/internal/processing" + "messenger/internal/server" +) + +const keyCleanupEvery = 10 * time.Minute + +func main() { + cfg, err := config.Load(os.LookupEnv) + if err != nil { + fmt.Fprintln(os.Stderr, "configuration error:", err) + os.Exit(2) + } + + alias := ident.NewAlias() + log := cfg.NewLogger().With("instance", alias) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := run(ctx, log, cfg, alias); err != nil { + log.Error("server stopped with error", "error", err.Error()) + os.Exit(1) + } +} + +// run connects the dependencies before it listens, so the port accepts +// connections only once they answer. +func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) error { + pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBMaxConns) + if err != nil { + return err + } + defer pool.Close() + sessions, err := valkey.NewClient(ctx, cfg.SessionStoreURL) + if err != nil { + return err + } + defer sessions.Close() + blobs, reconciler, err := newAttachmentStorage(pool, cfg.S3, os.LookupEnv) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", cfg.Addr()) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.Addr(), err) + } + defer ln.Close() + log.Info("starting", "env", cfg.Env, "addr", cfg.Addr(), + "gomaxprocs", runtime.GOMAXPROCS(0), "db_max_conns", cfg.DBMaxConns, + "drain_delay", cfg.DrainDelay.String(), "shutdown_timeout", cfg.ShutdownTimeout.String(), + "session_ttl", cfg.SessionTTL.String(), "idempotency_ttl", cfg.IdempotencyTTL.String()) + a := app.New(app.Deps{ + Log: log, + DB: pool, + Sessions: sessions, + SessionTTL: cfg.SessionTTL, + MaxBodyBytes: cfg.MaxBodyBytes, + InstanceID: alias, + IdempotencyTTL: cfg.IdempotencyTTL, + Blobs: blobs, + AttachmentsCfg: attachment.Config{ + BarrierDir: os.Getenv("LAB5_BARRIERS"), + MaxBytes: cfg.S3.MaxBytes, + MaxPixels: cfg.S3.MaxPixels, + }, + }) + var jobs config.Jobs + var barrier func(context.Context, pipeline.Job, string) error + if dir := os.Getenv("LAB5_BARRIERS"); dir != "" { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + barrier = processing.FileBarrier(dir) + } + if blobs != nil { + jobs, err = config.LoadJobs(os.LookupEnv) + if err != nil { + return err + } + } + var debugLn net.Listener + if cfg.DebugAddr != "" { + if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil { + return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) + } + publishDebugVars(alias, pool) + } + + ctx, cancel := context.WithCancel(ctx) + var background sync.WaitGroup + if blobs != nil { + background.Go(func() { processing.Dispatch(ctx, log, pipeline.NewStore(pool), jobs, barrier) }) + } + background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if reconciler != nil { + background.Go(func() { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + scan, cancel := context.WithTimeout(ctx, 2*time.Minute) + stats, err := reconciler.Run(scan, attachment.ReconcileOptions{Apply: true}, nil) + cancel() + if err != nil { + log.Warn("attachment cleanup incomplete", "error", err) + } else { + log.Info("attachment cleanup", "scanned_files", stats.ScannedFiles, "repaired", stats.Repaired) + } + } + } + }) + } + if debugLn != nil { + background.Go(func() { + if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { + log.Error("debug listener stopped", "error", err.Error()) + } + }) + } + + err = server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) + cancel() + background.Wait() + return err +} + +func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Lookup) (attachment.Blobs, *attachment.Reconciler, error) { + if cfg.Endpoint == "" { + return nil, nil, nil + } + blobs, err := s3store.New(cfg) + if err != nil { + return nil, nil, err + } + reconCfg, err := config.LoadS3(config.ReconcileS3(lookup)) + if err != nil { + return nil, nil, err + } + reconBlobs, err := s3store.New(reconCfg) + if err != nil { + return nil, nil, err + } + reconciler, err := attachment.NewReconcilerWithRetention(pool, reconBlobs, time.Hour, reconCfg.StagingRetention) + if err != nil { + return nil, nil, err + } + return blobs, reconciler, nil +} + +// publishDebugVars adds to expvar what the load tests read from a replica. +func publishDebugVars(alias string, pool *pgxpool.Pool) { + expvar.NewString("instance").Set(alias) + expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) })) + expvar.Publish("pgxpool", expvar.Func(func() any { + s := pool.Stat() + return map[string]any{ + "acquire_count": s.AcquireCount(), + "acquire_duration_ms": s.AcquireDuration().Milliseconds(), + "acquired_conns": s.AcquiredConns(), + "idle_conns": s.IdleConns(), + "total_conns": s.TotalConns(), + "max_conns": s.MaxConns(), + "empty_acquire_count": s.EmptyAcquireCount(), + "empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(), + "canceled_acquire_count": s.CanceledAcquireCount(), + "new_conns_count": s.NewConnsCount(), + } + })) +} diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go new file mode 100644 index 00000000..296607aa --- /dev/null +++ b/cmd/server/main_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "os" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +// A start-up failure after the background work began must still end the +// process with an error, not leave it waiting for a signal. +func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) { + if err := runWithBusyDebugAddr(t, 0); err == nil { + t.Fatal("run succeeded on a busy debug address") + } +} + +func TestFailedStartReleasesTheAPIPort(t *testing.T) { + probe, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := probe.Addr().(*net.TCPAddr).Port + probe.Close() + + if err := runWithBusyDebugAddr(t, port); err == nil { + t.Fatal("run succeeded on a busy debug address") + } + ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + t.Fatalf("the API port is still taken after the failed start: %v", err) + } + ln.Close() +} + +func runWithBusyDebugAddr(t *testing.T, port int) error { + t.Helper() + busy, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer busy.Close() + cfg := config.Config{ + Host: "127.0.0.1", + Port: port, + ShutdownTimeout: time.Second, + MaxBodyBytes: 1024, + DatabaseURL: db.DSN, + SessionStoreURL: kv.URL, + SessionTTL: time.Hour, + IdempotencyTTL: time.Hour, + DBMaxConns: 2, + DebugAddr: busy.Addr().String(), + } + done := make(chan error, 1) + go func() { + done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias") + }() + select { + case err := <-done: + return err + case <-time.After(10 * time.Second): + t.Fatal("run hung instead of returning the listen error") + return nil + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..9f264c52 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,193 @@ +// Command worker runs image consumers or an explicit operator action. +package main + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/broker" + "messenger/internal/imaging" + "messenger/internal/pipeline" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" + "messenger/internal/processing" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + if err := run(ctx, os.Args[1:]); err != nil { + slog.Error("worker stopped", "error", err) + os.Exit(1) + } +} + +const usage = "use failed, replay , reprocess , or duplicate " + +type command struct { + name, actor, barrierDir string + id uuid.UUID + version, attempt int +} + +func parseCommand(args []string) (command, error) { + var c command + if len(args) == 0 { + return c, nil + } + if dir, ok := strings.CutPrefix(args[0], "--barriers="); ok && len(args) == 1 && dir != "" { + c.barrierDir = dir + return c, nil + } + c.name = args[0] + switch c.name { + case "--prepare-large-png", "failed": + if len(args) == 1 { + return c, nil + } + return c, errors.New(usage) + case "replay", "reprocess": + if len(args) != 3 { + return c, errors.New(usage) + } + case "duplicate": + if len(args) != 4 { + return c, errors.New(usage) + } + default: + return c, errors.New(usage) + } + var err error + c.id, err = uuid.Parse(args[1]) + if err != nil { + return c, errors.New("invalid attachment UUID") + } + if c.name == "replay" { + c.actor = strings.TrimSpace(args[2]) + if c.actor == "" || len(c.actor) > 128 { + return c, errors.New("replay requires an operator name of up to 128 bytes") + } + return c, nil + } + c.version, err = strconv.Atoi(args[2]) + if err != nil || c.version < 1 { + return c, errors.New("invalid pipeline version") + } + if c.name == "duplicate" { + c.attempt, err = strconv.Atoi(args[3]) + if err != nil || c.attempt < 1 { + return c, errors.New("invalid attempt number") + } + } + return c, nil +} + +func run(ctx context.Context, args []string) error { + cmd, err := parseCommand(args) + if err != nil { + return err + } + if cmd.name == "--prepare-large-png" { + return imaging.PrepareLargePNG(os.Stdin) + } + if cmd.name != "" { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 15*time.Second) + defer cancel() + } + db, err := config.LoadDatabaseURL(os.LookupEnv) + if err != nil { + return err + } + pool, err := postgres.NewPool(ctx, db, 5) + if err != nil { + return err + } + defer pool.Close() + store := pipeline.NewStore(pool) + switch cmd.name { + case "failed": + jobs, err := store.Failed(ctx) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(jobs) + case "duplicate": + url, err := config.LoadBrokerURL(os.LookupEnv) + if err != nil { + return err + } + return duplicate(ctx, pool, url, cmd.id, cmd.version, cmd.attempt) + } + s3, err := config.LoadS3(os.LookupEnv) + if err != nil { + return err + } + blobs, err := s3store.New(s3) + if err != nil { + return err + } + exists := func(ctx context.Context, key string) (bool, error) { + _, err := blobs.Stat(ctx, key) + if errors.Is(err, attachment.ErrObjectMissing) { + return false, nil + } + return err == nil, err + } + switch cmd.name { + case "replay": + return store.Replay(ctx, cmd.id, cmd.actor, exists) + case "reprocess": + return store.Reprocess(ctx, cmd.id, cmd.version, exists) + } + cfg, err := config.LoadJobs(os.LookupEnv) + if err != nil { + return err + } + w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()} + if cmd.barrierDir != "" { + if err := os.MkdirAll(cmd.barrierDir, 0700); err != nil { + return err + } + w.Checkpoint = processing.FileBarrier(cmd.barrierDir) + } + if err := imaging.CleanScratch(); err != nil { + return err + } + w.Run(ctx) + return nil +} + +// duplicate sends a previously issued envelope through the actual broker. It +// cannot change SQL state or manufacture a future attempt. +func duplicate(ctx context.Context, pool *pgxpool.Pool, url string, id uuid.UUID, version, attempt int) error { + j := pipeline.Job{SchemaVersion: 1, Type: "attachment.process.v1", AttachmentID: id, PipelineVersion: version, Attempt: attempt} + err := pool.QueryRow(ctx, `SELECT job_id,created_at FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 AND $3::integer BETWEEN 1 AND attempt`, id, version, attempt).Scan(&j.JobID, &j.CreatedAt) + if err != nil { + return err + } + p, err := broker.NewPublisher(ctx, url) + if err != nil { + return err + } + defer p.Close() + body, err := json.Marshal(j) + if err != nil { + return err + } + return p.Publish(ctx, body) +} diff --git a/cmd/worker/main_test.go b/cmd/worker/main_test.go new file mode 100644 index 00000000..72ff8ad7 --- /dev/null +++ b/cmd/worker/main_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "messenger/internal/testenv" +) + +func TestInvalidCommandBeforeConfig(t *testing.T) { + t.Setenv("DATABASE_URL_FILE", "") + for _, args := range [][]string{{"unknown"}, {"replay", "bad-id", "admin"}, {"duplicate", "00000000-0000-4000-8000-000000000001", "1", "invalid"}} { + err := run(t.Context(), args) + if err == nil || strings.Contains(err.Error(), "DATABASE_URL") { + t.Errorf("run(%q) must report argument error, got %v", args, err) + } + } +} + +func TestFailedNeedsOnlyPostgres(t *testing.T) { + db, err := testenv.StartPostgres(t.Context()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + path := filepath.Join(t.TempDir(), "dsn") + if err := os.WriteFile(path, []byte(db.DSN), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("DATABASE_URL_FILE", path) + t.Setenv("BROKER_URL_FILE", "missing-broker") + t.Setenv("S3_ENDPOINT", "invalid") + t.Setenv("S3_SECRET_ACCESS_KEY_FILE", "") + if err := run(t.Context(), []string{"failed"}); err != nil { + t.Fatal(err) + } +} diff --git a/compose.bench.yaml b/compose.bench.yaml new file mode 100644 index 00000000..9c077168 --- /dev/null +++ b/compose.bench.yaml @@ -0,0 +1,47 @@ +# Load-test override of compose.yaml, used only by scripts/bench: every value +# an experiment varies, and the load generator. Defaults equal compose.yaml. +services: + lb: + environment: + LB_QUIET: ${LB_QUIET:-0} + API_MAXCONN: ${API_MAXCONN:-20} + deploy: + resources: + limits: + cpus: "${LB_CPUS:-0.5}" + + api: + environment: + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} + APP_DEBUG_ADDR: ":9090" + deploy: + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-256M} + + postgres: + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEMORY:-512M} + + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab5 + environment: + GOMEMLIMIT: 900MiB + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-2.0}" + memory: 1G diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 00000000..e5530c6c --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,13 @@ +# Explicit experiment overlay. Markers are local to the test worker. +services: + api: + environment: + JOB_MAX_ATTEMPTS: 5 + WORKER_CONCURRENCY: 2 + LAB5_BARRIERS: /tmp/lab5-faults + worker: + command: ["--barriers=/tmp/lab5-faults"] + environment: + JOB_MAX_ATTEMPTS: 5 + WORKER_CONCURRENCY: 2 + JOB_TIMEOUT_SECONDS: 15 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..8c4dddbf --- /dev/null +++ b/compose.yaml @@ -0,0 +1,345 @@ +# Local stand: HAProxy in front of interchangeable API replicas over shared +# PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data +# network has no way out. The load-test knobs are in compose.bench.yaml. +name: lab5 + +services: + lb: + image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa + depends_on: + api: + condition: service_healthy + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "${PUBLISH:-127.0.0.1:8080}:8080" + - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" + networks: + - edge + healthcheck: + # Through the balancer: `up --wait` returns once a replica is reachable. + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + api: + build: . + image: mtusi-messenger:lab5 + env_file: + - path: .env + required: false + environment: + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_STORE_URL_FILE: /run/secrets/session_store_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + # The background reconciler uses a separate identity with List rights. + S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id + S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key + # One decoder per replica. Progressive JPEG retains coefficient buffers; + # a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer. + GOMEMLIMIT: 1536MiB + secrets: + - database_url + - session_store_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + - s3_reconcile_access_key_id + - s3_reconcile_secret_access_key + networks: + - edge + - data + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_healthy + s3-init: + condition: service_completed_successfully + # Binary deadline 30s plus compensation, shutdown and drain margin. + stop_grace_period: 45s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 2G + + broker: + image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 + hostname: broker + secrets: + - source: rabbitmq_config + target: /etc/rabbitmq/conf.d/90-lab.conf + volumes: + - rabbitdata:/var/lib/rabbitmq + networks: [data] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 12 + deploy: + resources: + limits: + cpus: "1" + memory: 512M + + worker: + build: + context: . + target: worker + image: mtusi-messenger-worker:lab5 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "2" + memory: 2G + + migrate: + image: mtusi-messenger:lab5 + entrypoint: ["/usr/local/bin/migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + networks: + - data + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + # The image's check probes the API port, which migrate never opens. + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + postgres: + image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 + # 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql. + command: ["postgres", "-c", "max_connections=150"] + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + # PostgreSQL 18 keeps PGDATA in a versioned directory below this one. + volumes: + - pgdata:/var/lib/postgresql + networks: + - data + healthcheck: + # Over TCP: the first initialisation runs a temporary socket-only server. + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + valkey: + image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + secrets: + - valkey_auth + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + - valkeydata:/data + networks: + - data + healthcheck: + # valkey-cli exits 0 even on NOAUTH, so the reply itself is checked. + test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.5" + # maxmemory 256mb (valkey.conf) plus the server's own overhead. + memory: 320M + + s3: + image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0 + command: ["server", "-options=/etc/seaweedfs/lab.options"] + environment: + GOMEMLIMIT: 460MiB + secrets: + - s3_identities + volumes: + - ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro + # Both volume data and the image's /data/filerldb2 metadata live here. + - s3data:/data + networks: + - data + healthcheck: + # A reachable anonymous request must be refused, including before init. + test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + start_interval: 1s + stop_grace_period: 30s + pids_limit: 256 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + pids: 256 + + s3-init: + build: . + image: mtusi-messenger:lab5 + entrypoint: ["/usr/local/bin/s3init"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - source: s3_admin_access_key_id + target: s3_access_key_id + - source: s3_admin_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + s3: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + reconcile: + profiles: ["recon"] + build: . + image: mtusi-messenger:lab5 + entrypoint: ["/usr/local/bin/reconcile"] + environment: + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + GOMEMLIMIT: 100MiB + secrets: + - database_url + - source: s3_reconcile_access_key_id + target: s3_access_key_id + - source: s3_reconcile_secret_access_key + target: s3_secret_access_key + networks: + - data + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + +volumes: + rabbitdata: + pgdata: + valkeydata: + s3data: + +networks: + edge: {} + data: + internal: true + +secrets: + broker_url: + file: ./secrets/broker_url + rabbitmq_config: + file: ./secrets/rabbitmq.conf + database_url: + file: ./secrets/database_url + session_store_url: + file: ./secrets/session_store_url + postgres_password: + file: ./secrets/postgres_password + valkey_auth: + file: ./secrets/valkey_auth.conf + s3_identities: + file: ./secrets/s3_identities.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_reconcile_access_key_id: + file: ./secrets/s3_reconcile_access_key_id + s3_reconcile_secret_access_key: + file: ./secrets/s3_reconcile_secret_access_key diff --git a/evidence/codec-40mp.md b/evidence/codec-40mp.md new file mode 100644 index 00000000..60c12b69 --- /dev/null +++ b/evidence/codec-40mp.md @@ -0,0 +1,30 @@ +# Обработка файлов на границе 40 млн пикселей + +18 ноября 2026 г. Рабочее дерево на базе `276dbdad8a34`. Цель проверки — получить три читаемых варианта каждого допустимого исходника в пределах бюджета worker. + +## Воспроизведение + +Для генерации исходников использован Python с Pillow 11.1.0. Генератор записывает изображения и манифест с SHA-256 в игнорируемый каталог `bin/decoder-fixtures`. + +```sh +python -X utf8 -B scripts/lab4/generate-decoder-fixtures.py --output-dir bin/decoder-fixtures +make codec-test CODEC_STRESS=1 +``` + +Go 1.27.1, Docker Engine 28.5.1, Linux x86-64 в Docker Desktop, Intel Core i5-1240P. Контейнер теста использовал образ worker на Alpine 3.23 с ImageMagick 7.1.2-35 Q16-HDRI, 2 CPU и 2 GiB RAM, UID 10001. Запускался один тестовый процесс, без сети. Каждый файл обработан один раз, последовательно, без отдельного прогрева. Профиль v1 использует JPEG quality 85 и WebP quality 80. + +| Исходник | Байт на входе | Thumbnail, px / байт | Crop, px / байт | Optimized, px / байт | Время теста, с | +| --- | ---: | --- | --- | --- | ---: | +| Progressive RGB JPEG, 4:4:4, 8000×5000 | 469711 | 256×160 / 767 | 128×128 / 479 | 1280×800 / 1906 | 17,49 | +| Progressive CMYK JPEG, 4:4:4, 8000×5000 | 939079 | 256×160 / 526 | 128×128 / 382 | 1280×800 / 1900 | 24,15 | +| RGBA64 PNG, 40 000 000×1 | 1397354 | 256×1 / 169 | 128×128 / 225 | 1280×1 / 134 | 1,46 | + +Во всех трёх случаях тест декодировал оба выходных JPEG и WebP и сравнил фактические размеры с метаданными. Время включает чтение файла, обработку и проверочное декодирование внутри контейнера. HTTP, брокер и S3 в этом измерении не участвовали. Суммарный результат группы — PASS за 43,10 с. + +19 ноября на рабочем дереве после исправления координат crop и очистки временных файлов повторили тот же набор в Linux-контейнере с 2 CPU и 2 GiB. Все три теста прошли. Progressive RGB JPEG занял 9,78 с, progressive CMYK JPEG — 16,21 с, широкий RGBA64 PNG — 2,86 с. Измерения выполнены по одному разу на файл. + +Два свойства дополнительно проверили через публичный API. Для сплошного красного PNG с растром 320×200 и виртуальным холстом `2400×1800+850+30` в скачанном JPEG 128×128 получили RGB (254, 0, 0) в трёх точках по горизонтали. До исправления в центре был RGB (255, 255, 255). При обработке 40-мегапиксельного CMYK JPEG отправили `SIGKILL` процессу worker. В остановленном контейнере остался каталог попытки, в момент остановки измерили 82 680 KiB. После запуска того же контейнера этот каталог исчез. При штатном бюджете 60 с то же вложение перешло в `ready` на попытке 2 с тремя вариантами. После завершения в рабочем каталоге осталось 0 записей. В контрольном опыте до исправления 72 532 KiB пережили перезапуск. + +При отдельном запуске с укороченным бюджетом задания 30 с и параллельной нагрузкой рабочий каталог также очищался. На этом профиле CMYK-файл исчерпал пять попыток с `deadline_exceeded`. Для демонстрации восстановления предельного файла использовался штатный бюджет 60 с. + +В предварительном опыте обычный процесс ImageMagick получил SIGKILL на широкой PNG-строке при лимите контейнера 2 GiB. Теперь у PNG с осью длиннее 65 535 px отдельный процесс Go усредняет области для превью и берёт центральный квадрат из полного исходника. Такая работа занимает оба слота декодера до выпуска результатов. На JPEG действует лимит одного потока ImageMagick на задание, 96 MiB для пиксельного кэша, отключённый mmap и до 2 GiB дискового кэша. Общий дедлайн преобразования — 45 секунд. Измерение подтверждает прохождение перечисленных файлов под лимитом контейнера. Пиковый RSS в этом запуске не измерялся. diff --git a/evidence/go-race.txt b/evidence/go-race.txt new file mode 100644 index 00000000..a736bbe7 --- /dev/null +++ b/evidence/go-race.txt @@ -0,0 +1,37 @@ +go test -race -p 2 -count=1 ./... + +? messenger/cmd/migrate [no test files] +ok messenger/cmd/reconcile 1.829s +? messenger/cmd/s3init [no test files] +ok messenger/cmd/server 14.169s +? messenger/cmd/worker [no test files] +ok messenger/internal/app 120.381s +ok messenger/internal/attachment 192.997s +? messenger/internal/attachment/store [no test files] +ok messenger/internal/auth 20.092s +? messenger/internal/auth/store [no test files] +ok messenger/internal/broker 52.153s +ok messenger/internal/chat 19.186s +ok messenger/internal/chat/store 18.159s +? messenger/internal/health [no test files] +ok messenger/internal/imaging 3.603s +ok messenger/internal/pipeline 19.592s +ok messenger/internal/platform/apperr 4.316s +ok messenger/internal/platform/config 3.654s +ok messenger/internal/platform/faultbarrier 2.507s +ok messenger/internal/platform/httpx 1.710s +ok messenger/internal/platform/ident 2.617s +ok messenger/internal/platform/pagination 2.899s +ok messenger/internal/platform/password 4.071s +ok messenger/internal/platform/postgres 15.192s +ok messenger/internal/platform/s3store 224.080s +ok messenger/internal/platform/valkey 7.521s +ok messenger/internal/processing 5.820s +ok messenger/internal/server 13.380s +ok messenger/internal/testenv 3.811s +ok messenger/internal/user 21.605s +? messenger/internal/user/store [no test files] +? messenger/migrations [no test files] +ok messenger/tools/concurrency 23.363s +ok messenger/tools/loadgen 17.699s +ok messenger/tools/reconcilebench 1.482s diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt new file mode 100644 index 00000000..648d6fce --- /dev/null +++ b/evidence/public-checks.txt @@ -0,0 +1,84 @@ +2026-11-18 | Docker Compose project lab5 | normal configuration | http://localhost:18085 + +make check PYTHON=python +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +make validate PYTHON=.venv/Scripts/python.exe +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +make test PYTHON=python BASE_URL=http://localhost:18085 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.cmd BASE_URL=http://localhost:18085 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +make test-failover PYTHON=python ACTION_SCRIPT=scripts/stop-one.cmd BASE_URL=http://localhost:18085 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 8.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +docker compose start api +sh scripts/wait-ready.sh +balancer readiness confirmed + +2026-11-19 | after review | isolated Compose project lab5-review | normal configuration | http://localhost:19085 + +make test PYTHON=python +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.cmd +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +make test-failover PYTHON=python ACTION_SCRIPT=scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 0, "first_recovery_seconds": 6.369} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +2026-11-19 | working tree after teamlead fixes on c6a5e9e | isolated Compose project lab5-fix-check | http://localhost:19285 + +go build ./... && go vet ./... && go test -race -p 2 -count=1 ./... +Exit code 0. Additional PostgreSQL regressions for attachment, chat/store and pipeline passed with -race. + +make check validate PYTHON=.venv/Scripts/python.exe +Exit code 0. +python -B -m unittest discover -s scripts/lab4 +15 tests, OK. + +make track2 PYTHON=python +External WORKER_CONCURRENCY=1 and JOB_MAX_ATTEMPTS=1. Complete, 11 scenarios, 479.143 seconds. See track2/run-after-fixes.json. + +Normal Compose profile restored before the following public checks. +make test PYTHON=python +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +make upload-memory PYTHON=python +Exit code 0. 10/10 responses with HTTP 202 for each of 1 MiB and 10 MiB. Both control downloads matched. See upload-memory.json. + +make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.cmd +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +make test-failover PYTHON=python ACTION_SCRIPT=scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 6.168} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +Both API replicas restored. Worker-off defense sequence repeated through HTTP. See worker-off-after-fixes.json. + +2026-11-20 | pre-submission check | fresh Compose project lab5-submit-nov20 | normal configuration + +go build ./... +go vet ./... +go test -race -p 2 -count=1 ./... +All three commands exited with code 0. Tested application and scripts correspond to b05b338. + +make check validate PYTHON=.venv/Scripts/python.exe +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +python -B -m unittest discover -s scripts/lab4 +Ran 15 tests. OK. + +make test PYTHON=python BASE_URL=http://127.0.0.1:32768 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +make test-failover PYTHON=python BASE_URL=http://127.0.0.1:32768 ACTION_SCRIPT=scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 0, "first_recovery_seconds": 7.324} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/track2/run-1.json b/evidence/track2/run-1.json new file mode 100644 index 00000000..dd80581c --- /dev/null +++ b/evidence/track2/run-1.json @@ -0,0 +1,450 @@ +{ + "project": "lab5", + "complete": true, + "scenarios": { + "staging_before_commit": { + "unaccepted": "01a1ee52-1be9-7d4e-9b16-0eb89161e76e", + "source_sha256": "987a16861654a3d02b3525c449dfdaf61f06fa7f751c2182c12ae8a9587be790", + "attachment_rows": 0 + }, + "commit_before_publish": { + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "61d2468d-e79b-4a86-8ef2-48d132bd17e2", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "message": "01a1ee52-4354-7e75-b7eb-eef5cf84f3cf", + "after": { + "attachment": "01a1ee52-3f2d-719f-bb9b-86f39a6d261e", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "61d2468d-e79b-4a86-8ef2-48d132bd17e2", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "broker_confirm_before_sql": { + "broker_confirm_before_sql_commit": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "a3c0457e-37f1-4562-afa7-f1c771e159f7", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "confirmed_job_before_broker_restart": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "a3c0457e-37f1-4562-afa7-f1c771e159f7", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "broker_messages_before_restart": 2, + "broker_messages_after_restart": 2, + "after": { + "attachment": "01a1ee5c-8227-733d-a0c4-ada381ddcbfe", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "a3c0457e-37f1-4562-afa7-f1c771e159f7", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_claimed": { + "phase": "claimed", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "11f5390b-6f23-4584-bceb-3dc712575eb3", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": null, + "optimized": null, + "thumbnail": null + }, + "after": { + "attachment": "01a1ee54-2f01-7e6d-95c1-9791b83a3b20", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "11f5390b-6f23-4584-bceb-3dc712575eb3", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_objects": { + "phase": "objects", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "fab4466e-899e-441f-ada7-06297a5c4ab1", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1ee55-2f6a-742c-8740-d202f49f4395", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "fab4466e-899e-441f-ada7-06297a5c4ab1", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_committed": { + "phase": "committed", + "before": { + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "5bc84f9f-bbde-43ea-ad1b-94270742c6c3", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1ee56-1caf-799f-bda3-174c0f6580b1", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "5bc84f9f-bbde-43ea-ad1b-94270742c6c3", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "same_version_lease_takeover": { + "first_held": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "200fd37a-abb3-491d-851b-6e4169d2d2d3", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "second_committed": { + "attachment": "01a1ee57-165b-71b8-b1b0-f3dbb0f01b94", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "200fd37a-abb3-491d-851b-6e4169d2d2d3", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_first_ack": { + "attachment": "01a1ee57-165b-71b8-b1b0-f3dbb0f01b94", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "200fd37a-abb3-491d-851b-6e4169d2d2d3", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "parallel_duplicate_and_reorder": { + "reference_v1_webp": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "before_releasing_v1": { + "attachment": "01a1ee58-4352-7554-a610-3ab6740bc55b", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "9f933d9a-04e1-4e9c-9168-f500e979cc0c", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "6fd216b3-007f-4533-a65a-092ef8d93fd0", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_acks_in_order": [ + 2, + 1 + ], + "after": { + "attachment": "01a1ee58-4352-7554-a610-3ab6740bc55b", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "9f933d9a-04e1-4e9c-9168-f500e979cc0c", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "6fd216b3-007f-4533-a65a-092ef8d93fd0", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "retry_exhaustion_replay": { + "injection": "absent source key assigned only to synthetic job", + "exhausted": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "796b4363-d6c8-413b-9fb9-6fc4e8a2af64", + "attempt": 5, + "state": "failed", + "error": "source_missing", + "confirmed": false + } + ], + "variant_count": 0 + }, + "after": { + "attachment": "01a1ee59-5ad8-704d-bf39-14acac47bfcf", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "796b4363-d6c8-413b-9fb9-6fc4e8a2af64", + "attempt": 6, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "audit_rows": 1 + }, + "permanent_failure": { + "injection": "different valid original assigned only to synthetic job", + "failed": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "2e5cd010-5f77-4610-b448-1f2b3d04777a", + "attempt": 1, + "state": "failed", + "error": "source_mismatch", + "confirmed": false + } + ], + "variant_count": 0 + } + }, + "unaccepted_staging_cleanup": { + "before": { + "kind": "stale_staging", + "key": "staging/01a1ee52-1be9-7d4e-9b16-0eb89161e76e", + "action": "delete_object", + "applied": false + }, + "applied": { + "kind": "stale_staging", + "key": "staging/01a1ee52-1be9-7d4e-9b16-0eb89161e76e", + "action": "delete_object", + "applied": true + }, + "after_findings": 0 + } + }, + "started_at": "2026-11-18T11:01:59.371482+00:00", + "finished_at": "2026-11-18T11:15:57.692241+00:00", + "original_matrix_seconds": 629.758, + "additional_broker_volume_check_seconds": 159.97, + "elapsed_seconds": 789.728, + "timing_method": "Initial matrix file creation to completion plus timed rerun of the publisher-crash scenario with an additional broker restart" +} diff --git a/evidence/track2/run-2.json b/evidence/track2/run-2.json new file mode 100644 index 00000000..92a256ab --- /dev/null +++ b/evidence/track2/run-2.json @@ -0,0 +1,460 @@ +{ + "project": "lab5", + "complete": true, + "scenarios": { + "staging_before_commit": { + "unaccepted": "01a1ee5f-0e26-7596-b66e-ba780d524b2d", + "source_sha256": "467a81323f47737ba40b911609899e1daf2498ebefd5e6b26b0eab288484c38e", + "attachment_rows": 0 + }, + "commit_before_publish": { + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "e6623c35-9963-4d3e-9e3e-9b424ce30924", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "message": "01a1ee5f-3bdd-7752-8c2a-b1356814254f", + "after": { + "attachment": "01a1ee5f-3709-7178-9527-1c0651c4b6e0", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "e6623c35-9963-4d3e-9e3e-9b424ce30924", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "broker_confirm_before_sql": { + "broker_confirm_before_sql_commit": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "89a6f7a2-a95c-4ba4-bd41-e353908e68d9", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "confirmed_job_before_broker_restart": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "89a6f7a2-a95c-4ba4-bd41-e353908e68d9", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "broker_messages_before_restart": 2, + "broker_messages_after_restart": 2, + "after": { + "attachment": "01a1ee60-23c1-7779-bd6c-a09ddf5f307c", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "89a6f7a2-a95c-4ba4-bd41-e353908e68d9", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_claimed": { + "phase": "claimed", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "f38497e6-36f7-4595-b82a-447521566009", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": null, + "optimized": null, + "thumbnail": null + }, + "after": { + "attachment": "01a1ee62-5a5b-7ab4-b1ab-2dfa58b8ad9e", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "f38497e6-36f7-4595-b82a-447521566009", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_objects": { + "phase": "objects", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "84e530d3-daef-4450-8143-19b6a92b7fa1", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1ee63-6c96-7c2f-a9ee-7d1f1c3a4a6c", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "84e530d3-daef-4450-8143-19b6a92b7fa1", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_committed": { + "phase": "committed", + "before": { + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "d95c1517-95da-4aa6-b8e6-4bd3355d50be", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1ee64-7147-747c-a291-62709e8da46d", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "d95c1517-95da-4aa6-b8e6-4bd3355d50be", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "same_version_lease_takeover": { + "first_held": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "50ffa4a8-b441-4e4b-b098-2f304bb7bc03", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "second_committed": { + "attachment": "01a1ee65-3c0c-7e07-884c-b15419605298", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "50ffa4a8-b441-4e4b-b098-2f304bb7bc03", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_first_ack": { + "attachment": "01a1ee65-3c0c-7e07-884c-b15419605298", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "50ffa4a8-b441-4e4b-b098-2f304bb7bc03", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "parallel_duplicate_and_reorder": { + "reference_v1_webp": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "before_releasing_v1": { + "attachment": "01a1ee66-255e-70b4-8093-a4f2cc21d4d9", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "5a50ccea-26a3-4288-97d0-614be02e1ae4", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "f3575b8d-d0cb-4d31-9ffe-f326e8b4840d", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_acks_in_order": [ + 2, + 1 + ], + "after": { + "attachment": "01a1ee66-255e-70b4-8093-a4f2cc21d4d9", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "5a50ccea-26a3-4288-97d0-614be02e1ae4", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "f3575b8d-d0cb-4d31-9ffe-f326e8b4840d", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "retry_exhaustion_replay": { + "injection": "absent source key assigned only to synthetic job", + "exhausted": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "50fa2497-8c8b-4898-8f53-bc767beace3d", + "attempt": 5, + "state": "failed", + "error": "source_missing", + "confirmed": false + } + ], + "variant_count": 0 + }, + "after": { + "attachment": "01a1ee66-c999-7ab5-b3e6-02504f21f7d1", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "50fa2497-8c8b-4898-8f53-bc767beace3d", + "attempt": 6, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "audit_rows": 1 + }, + "permanent_failure": { + "injection": "different valid original assigned only to synthetic job", + "failed": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "ceab5fc6-4d4a-409a-956d-93303edec86e", + "attempt": 1, + "state": "failed", + "error": "source_mismatch", + "confirmed": false + } + ], + "variant_count": 0 + } + }, + "unaccepted_staging_cleanup": { + "before": { + "kind": "stale_staging", + "key": "staging/01a1ee5f-0e26-7596-b66e-ba780d524b2d", + "action": "delete_object", + "applied": false + }, + "applied": { + "kind": "stale_staging", + "key": "staging/01a1ee5f-0e26-7596-b66e-ba780d524b2d", + "action": "delete_object", + "applied": true + }, + "after_findings": 0 + } + }, + "seconds": { + "staging_before_commit": 9.621, + "commit_before_publish": 62.648, + "broker_confirm_before_sql": 144.738, + "worker_crash_claimed": 70.598, + "worker_crash_objects": 66.407, + "worker_crash_committed": 52.276, + "same_version_lease_takeover": 60.688, + "parallel_duplicate_and_reorder": 41.799, + "retry_exhaustion_replay": 30.308, + "permanent_failure": 26.619, + "unaccepted_staging_cleanup": 6.511 + }, + "started_at": "2026-11-18T11:16:05.212539+00:00", + "finished_at": "2026-11-18T11:25:37.442245+00:00", + "elapsed_seconds": 572.23 +} diff --git a/evidence/track2/run-after-fixes.json b/evidence/track2/run-after-fixes.json new file mode 100644 index 00000000..254b9600 --- /dev/null +++ b/evidence/track2/run-after-fixes.json @@ -0,0 +1,465 @@ +{ + "project": "lab5-fix-check", + "complete": true, + "scenarios": { + "staging_before_commit": { + "unaccepted": "01a1f3d2-1663-703b-ba43-6e2ba2bf4ca1", + "source_sha256": "7dbb3fb09f66d46d436d2f15cfa0e6e2b1b5e26ca8cb3e49c129e3dcbc67dd08", + "attachment_rows": 0 + }, + "commit_before_publish": { + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "3d8068d6-cf1e-406e-aa34-8cdfe474316b", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "message": "01a1f3d2-3483-7cf8-98f4-c248e6373686", + "after": { + "attachment": "01a1f3d2-30fc-76a6-ab63-af8263e0932c", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "3d8068d6-cf1e-406e-aa34-8cdfe474316b", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "broker_confirm_before_sql": { + "broker_confirm_before_sql_commit": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "6ea1bcd2-283f-490c-85dc-3387a840b5e9", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "confirmed_job_before_broker_restart": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "6ea1bcd2-283f-490c-85dc-3387a840b5e9", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "message_while_publisher_held": { + "id": "01a1f3d3-b3dc-7745-aa9e-ee041d90110f", + "status": 201, + "seconds": 0.147 + }, + "broker_messages_before_restart": 2, + "broker_messages_after_restart": 2, + "after": { + "attachment": "01a1f3d2-dd56-73e6-a53c-ff78f4488df9", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "6ea1bcd2-283f-490c-85dc-3387a840b5e9", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_claimed": { + "phase": "claimed", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "48b90aa4-670e-4774-bdc6-cb068c1a0b14", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": null, + "optimized": null, + "thumbnail": null + }, + "after": { + "attachment": "01a1f3d5-101d-7362-8e8f-047b587a0e3d", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "48b90aa4-670e-4774-bdc6-cb068c1a0b14", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_objects": { + "phase": "objects", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "8b84239f-33b5-4dec-bce5-5bd1d670c5f8", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1f3d6-0d97-7114-8882-94ffc8c062cf", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "8b84239f-33b5-4dec-bce5-5bd1d670c5f8", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_committed": { + "phase": "committed", + "before": { + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "2be611d2-9d53-4452-a7ae-c2e617e48b45", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1f3d6-e20d-738e-a91c-75222aa70cf5", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "2be611d2-9d53-4452-a7ae-c2e617e48b45", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "same_version_lease_takeover": { + "first_held": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "0a80fdb6-8c01-43a0-afa4-ea05fe970f59", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "second_committed": { + "attachment": "01a1f3d7-69e7-7505-b4e7-f1cde135696e", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "0a80fdb6-8c01-43a0-afa4-ea05fe970f59", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_first_ack": { + "attachment": "01a1f3d7-69e7-7505-b4e7-f1cde135696e", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "0a80fdb6-8c01-43a0-afa4-ea05fe970f59", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "parallel_duplicate_and_reorder": { + "reference_v1_webp": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "before_releasing_v1": { + "attachment": "01a1f3d8-33a1-7cca-8bfe-21981c9bfbff", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "8179f193-e5af-424a-b904-cd94806b8cf9", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "fe77b49d-b01c-49d6-9d08-a08c07cf8a8d", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_acks_in_order": [ + 2, + 1 + ], + "after": { + "attachment": "01a1f3d8-33a1-7cca-8bfe-21981c9bfbff", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "8179f193-e5af-424a-b904-cd94806b8cf9", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "fe77b49d-b01c-49d6-9d08-a08c07cf8a8d", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "retry_exhaustion_replay": { + "injection": "absent source key assigned only to synthetic job", + "exhausted": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "4e615206-5278-414f-8f71-d84971e97d3a", + "attempt": 5, + "state": "failed", + "error": "source_missing", + "confirmed": false + } + ], + "variant_count": 0 + }, + "after": { + "attachment": "01a1f3d8-926f-7684-bf62-8289232638cb", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "4e615206-5278-414f-8f71-d84971e97d3a", + "attempt": 6, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "audit_rows": 1 + }, + "permanent_failure": { + "injection": "different valid original assigned only to synthetic job", + "failed": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "89899dd3-bd2d-40c1-ab9b-69ff3d5655db", + "attempt": 1, + "state": "failed", + "error": "source_mismatch", + "confirmed": false + } + ], + "variant_count": 0 + } + }, + "unaccepted_staging_cleanup": { + "before": { + "kind": "stale_staging", + "key": "staging/01a1f3d2-1663-703b-ba43-6e2ba2bf4ca1", + "action": "delete_object", + "applied": false + }, + "applied": { + "kind": "stale_staging", + "key": "staging/01a1f3d2-1663-703b-ba43-6e2ba2bf4ca1", + "action": "delete_object", + "applied": true + }, + "after_findings": 0 + } + }, + "seconds": { + "staging_before_commit": 5.621, + "commit_before_publish": 43.92, + "broker_confirm_before_sql": 143.672, + "worker_crash_claimed": 65.682, + "worker_crash_objects": 53.866, + "worker_crash_committed": 35.637, + "same_version_lease_takeover": 51.139, + "parallel_duplicate_and_reorder": 25.06, + "retry_exhaustion_replay": 37.585, + "permanent_failure": 13.202, + "unaccepted_staging_cleanup": 3.749 + }, + "started_at": "2026-11-19T12:39:52.840388+00:00", + "finished_at": "2026-11-19T12:47:51.983061+00:00", + "elapsed_seconds": 479.143 +} diff --git a/evidence/track2/run-review.json b/evidence/track2/run-review.json new file mode 100644 index 00000000..abd54b2f --- /dev/null +++ b/evidence/track2/run-review.json @@ -0,0 +1,460 @@ +{ + "project": "lab5-review", + "complete": true, + "scenarios": { + "staging_before_commit": { + "unaccepted": "01a1f2e1-533a-7e57-a9c1-e2e26acd0ccb", + "source_sha256": "c1174ee7fa898e6f07fc00b4e7f28a9643573abc1c623fe7b4d15444efcf48ed", + "attachment_rows": 0 + }, + "commit_before_publish": { + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "e5a44dfc-45cf-4dd0-a68c-179eda00e24a", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "message": "01a1f2e1-71ca-72e8-8200-b241790369cc", + "after": { + "attachment": "01a1f2e1-6e6d-77db-91c8-5fd2cbb27501", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "e5a44dfc-45cf-4dd0-a68c-179eda00e24a", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "broker_confirm_before_sql": { + "broker_confirm_before_sql_commit": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "943b8892-b76e-47c2-9b1e-534e15191d9a", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": false + } + ], + "variant_count": 0 + }, + "confirmed_job_before_broker_restart": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "943b8892-b76e-47c2-9b1e-534e15191d9a", + "attempt": 1, + "state": "queued", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "broker_messages_before_restart": 2, + "broker_messages_after_restart": 2, + "after": { + "attachment": "01a1f2e1-e993-73a4-9979-373bcad70aaa", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "943b8892-b76e-47c2-9b1e-534e15191d9a", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_claimed": { + "phase": "claimed", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "5b2bfe20-356f-4d4e-b40c-4d13d09c4546", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": null, + "optimized": null, + "thumbnail": null + }, + "after": { + "attachment": "01a1f2e3-431b-7147-a44b-90c282e2c8aa", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "5b2bfe20-356f-4d4e-b40c-4d13d09c4546", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_objects": { + "phase": "objects", + "before": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "34267a34-01d1-4714-970e-adfa1d010e29", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1f2e4-138b-74b1-a61d-7535491bab2f", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "34267a34-01d1-4714-970e-adfa1d010e29", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "worker_crash_committed": { + "phase": "committed", + "before": { + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "6cd1d4b0-d54c-4ddf-b218-374b6cb3d145", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "storage_sha256_at_barrier": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "after": { + "attachment": "01a1f2e4-e44b-71a6-a2f0-1428ee721758", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "6cd1d4b0-d54c-4ddf-b218-374b6cb3d145", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "same_version_lease_takeover": { + "first_held": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "417120a3-4f24-4294-a0b0-a7858924b69c", + "attempt": 1, + "state": "processing", + "error": null, + "confirmed": true + } + ], + "variant_count": 0 + }, + "second_committed": { + "attachment": "01a1f2e5-4ebd-780d-b531-e98c55bc4b3c", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "417120a3-4f24-4294-a0b0-a7858924b69c", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_first_ack": { + "attachment": "01a1f2e5-4ebd-780d-b531-e98c55bc4b3c", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "417120a3-4f24-4294-a0b0-a7858924b69c", + "attempt": 2, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "parallel_duplicate_and_reorder": { + "reference_v1_webp": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "before_releasing_v1": { + "attachment": "01a1f2e6-2afe-7dc8-b789-fb3f203da96a", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "c8ad5936-e488-47c0-919a-688b892bc9e2", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "073f2a9b-ba29-4e92-a47b-914d0b964b78", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "after_acks_in_order": [ + 2, + 1 + ], + "after": { + "attachment": "01a1f2e6-2afe-7dc8-b789-fb3f203da96a", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "b99d07b62b1b83e732691b1895a5fd863be0def9a17e2bc8ebd1b315db2cd29c", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 2, + "published": 2, + "jobs": [ + { + "version": 1, + "id": "c8ad5936-e488-47c0-919a-688b892bc9e2", + "attempt": 1, + "state": "superseded", + "error": null, + "confirmed": true + }, + { + "version": 2, + "id": "073f2a9b-ba29-4e92-a47b-914d0b964b78", + "attempt": 1, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + } + }, + "retry_exhaustion_replay": { + "injection": "absent source key assigned only to synthetic job", + "exhausted": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "1372a91e-858e-4feb-b031-cd4943ab9d22", + "attempt": 5, + "state": "failed", + "error": "source_missing", + "confirmed": false + } + ], + "variant_count": 0 + }, + "after": { + "attachment": "01a1f2e6-a686-7a62-a0e1-09d0f9cf6e66", + "original_sha256": "c7a60862e32d552fadd3c10b2a1d44886999cabbcfc3147fe947873fa5b143af", + "variant_sha256": { + "crop": "2ba9750989e7fde217ad5b00ee18fbd73d9278a54bf7d7165c40096297a2f1f2", + "optimized": "544068e10875fc2efd8e694e90db90b782ae717b618d2efe30393f3df55e23ca", + "thumbnail": "6b253f725ffaf7bfde827edfb71f328c38ffec4a4147211bedba5115a776f5a7" + }, + "target": 1, + "published": 1, + "jobs": [ + { + "version": 1, + "id": "1372a91e-858e-4feb-b031-cd4943ab9d22", + "attempt": 6, + "state": "ready", + "error": null, + "confirmed": true + } + ], + "variant_count": 3 + }, + "audit_rows": 1 + }, + "permanent_failure": { + "injection": "different valid original assigned only to synthetic job", + "failed": { + "target": 1, + "published": null, + "jobs": [ + { + "version": 1, + "id": "b9dd9972-a5f0-4183-8aba-aab629b7189d", + "attempt": 1, + "state": "failed", + "error": "source_mismatch", + "confirmed": false + } + ], + "variant_count": 0 + } + }, + "unaccepted_staging_cleanup": { + "before": { + "kind": "stale_staging", + "key": "staging/01a1f2e1-533a-7e57-a9c1-e2e26acd0ccb", + "action": "delete_object", + "applied": false + }, + "applied": { + "kind": "stale_staging", + "key": "staging/01a1f2e1-533a-7e57-a9c1-e2e26acd0ccb", + "action": "delete_object", + "applied": true + }, + "after_findings": 0 + } + }, + "seconds": { + "staging_before_commit": 6.106, + "commit_before_publish": 31.334, + "broker_confirm_before_sql": 88.361, + "worker_crash_claimed": 54.078, + "worker_crash_objects": 53.264, + "worker_crash_committed": 27.227, + "same_version_lease_takeover": 53.318, + "parallel_duplicate_and_reorder": 34.439, + "retry_exhaustion_replay": 27.452, + "permanent_failure": 19.796, + "unaccepted_staging_cleanup": 4.568 + }, + "started_at": "2026-11-19T08:16:54.478259+00:00", + "finished_at": "2026-11-19T08:23:34.338852+00:00", + "elapsed_seconds": 399.954 +} diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..ed3cea06 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,1558 @@ +{ + "schema_version": 1, + "started_at": "2026-11-19T12:48:42.829424+00:00", + "ok": true, + "driver": { + "os": "Windows", + "architecture": "AMD64", + "logical_cpus": 16, + "python": "3.13.1" + }, + "measurement": { + "source": "docker stats JSON stream", + "cadence": "Docker-managed, typically about 1 second", + "limitation": "sampled peaks may miss sub-second maxima; includes post-upload sampling tail", + "memory_semantics": "Docker CLI memory usage (working set, cache excluded where supported)", + "baseline_seconds": 3, + "tail_seconds": 3, + "same_bytes_per_case": true, + "p95_method": "nearest rank", + "profile": "standard", + "png_compression": "zlib level 1; random RGB; npAD padding to exact byte limit" + }, + "cases": [ + { + "name": "small", + "width": 512, + "height": 512, + "seed": 41, + "payload_bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "bit_depth": 8, + "color_type": 2, + "nominal_decoded_bytes": 1048576, + "requests": 10, + "attempted_bytes": 10485760, + "successful_bytes": 10485760, + "accepted": 10, + "status_counts": { + "202": 10 + }, + "errors": 0, + "duration_s": 0.356045, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 320.832, + "p95_ms": 352.175, + "max_ms": 352.175 + }, + "memory": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "baseline_bytes": 67381494, + "sampled_peak_bytes": 68692214, + "delta_bytes": 1310720, + "baseline_samples": 14, + "load_samples": 12 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "baseline_bytes": 67979182, + "sampled_peak_bytes": 71428997, + "delta_bytes": 3449815, + "baseline_samples": 14, + "load_samples": 12 + } + }, + "memory_samples": [ + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 1.56275, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 68105011, + "limit_bytes": 2147483648, + "elapsed_s": 1.562815, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 1.562836, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 68105011, + "limit_bytes": 2147483648, + "elapsed_s": 1.562851, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 2.061795, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 68105011, + "limit_bytes": 2147483648, + "elapsed_s": 2.061836, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 2.061857, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 68105011, + "limit_bytes": 2147483648, + "elapsed_s": 2.061872, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 2.561653, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 2.561688, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 2.561706, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 2.561722, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 3.061606, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 3.061635, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 3.061649, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 3.06166, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67507323, + "limit_bytes": 2147483648, + "elapsed_s": 3.561973, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 3.562004, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67507323, + "limit_bytes": 2147483648, + "elapsed_s": 3.562019, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 3.562031, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67507323, + "limit_bytes": 2147483648, + "elapsed_s": 4.062157, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 4.062195, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67507323, + "limit_bytes": 2147483648, + "elapsed_s": 4.062214, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67968696, + "limit_bytes": 2147483648, + "elapsed_s": 4.06223, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 4.561945, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67979182, + "limit_bytes": 2147483648, + "elapsed_s": 4.56198, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 4.561997, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67979182, + "limit_bytes": 2147483648, + "elapsed_s": 4.56201, + "case": "small", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 5.061665, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67979182, + "limit_bytes": 2147483648, + "elapsed_s": 5.061707, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 67381494, + "limit_bytes": 2147483648, + "elapsed_s": 5.061728, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 67979182, + "limit_bytes": 2147483648, + "elapsed_s": 5.061746, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 5.561672, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71397540, + "limit_bytes": 2147483648, + "elapsed_s": 5.561783, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 5.561822, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71397540, + "limit_bytes": 2147483648, + "elapsed_s": 5.561841, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 6.062033, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71397540, + "limit_bytes": 2147483648, + "elapsed_s": 6.062073, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 6.062091, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71397540, + "limit_bytes": 2147483648, + "elapsed_s": 6.062107, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 6.561956, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71418511, + "limit_bytes": 2147483648, + "elapsed_s": 6.56203, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 6.562068, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71418511, + "limit_bytes": 2147483648, + "elapsed_s": 6.562104, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 7.062347, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71418511, + "limit_bytes": 2147483648, + "elapsed_s": 7.062462, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 7.062492, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71418511, + "limit_bytes": 2147483648, + "elapsed_s": 7.062506, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68692214, + "limit_bytes": 2147483648, + "elapsed_s": 7.56194, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71428997, + "limit_bytes": 2147483648, + "elapsed_s": 7.561969, + "case": "small", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68692214, + "limit_bytes": 2147483648, + "elapsed_s": 7.561984, + "case": "small", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71428997, + "limit_bytes": 2147483648, + "elapsed_s": 7.561996, + "case": "small", + "phase": "load" + } + ], + "memory_error": null, + "results": [ + { + "index": 0, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-3ced-7396-ab34-58316a391de9", + "latency_ms": 253.894, + "start_offset_ms": 1.444 + }, + { + "index": 1, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-3d03-7b7c-ac9e-481d265064a1", + "latency_ms": 348.027, + "start_offset_ms": 0.096 + }, + { + "index": 2, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-3cec-72a7-83fe-e6b40e264507", + "latency_ms": 173.804, + "start_offset_ms": 0.312 + }, + { + "index": 3, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-3cfe-7005-963c-936696721843", + "latency_ms": 317.056, + "start_offset_ms": 0.495 + }, + { + "index": 4, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-3cec-70f4-a14d-ccac7e347c78", + "latency_ms": 160.06, + "start_offset_ms": 0.677 + }, + { + "index": 5, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-3d00-75b7-8931-374fa44a5ad8", + "latency_ms": 339.821, + "start_offset_ms": 1.655 + }, + { + "index": 6, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-3cfd-735c-acd8-699a3ea2e588", + "latency_ms": 324.608, + "start_offset_ms": 1.836 + }, + { + "index": 7, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-3cf7-7789-9d4b-16b04bce4150", + "latency_ms": 334.025, + "start_offset_ms": 0.977 + }, + { + "index": 8, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-3cf2-752b-a4cb-8ef95eaf7eef", + "latency_ms": 259.974, + "start_offset_ms": 1.185 + }, + { + "index": 9, + "status": 202, + "ok": true, + "request_bytes": 1048576, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-3d09-7e54-9912-a8c12b664dde", + "latency_ms": 352.175, + "start_offset_ms": 2.149 + } + ], + "download": { + "ok": true, + "bytes": 1048576, + "sha256": "571457cffcc868ecd536880fdc589bd362542c4645c1698402af4c1a04c71079", + "status": 200 + }, + "state_before": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "state_after": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "ok": true + }, + { + "name": "limit", + "width": 1600, + "height": 1600, + "seed": 42, + "payload_bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "bit_depth": 8, + "color_type": 2, + "nominal_decoded_bytes": 10240000, + "requests": 10, + "attempted_bytes": 104857600, + "successful_bytes": 104857600, + "accepted": 10, + "status_counts": { + "202": 10 + }, + "errors": 0, + "duration_s": 3.1045, + "observed_max_inflight": 10, + "distinct_attachment_ids": 10, + "latency": { + "p50_ms": 2637.8559999999998, + "p95_ms": 3102.734, + "max_ms": 3102.734 + }, + "memory": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "baseline_bytes": 68681728, + "sampled_peak_bytes": 68755128, + "delta_bytes": 73400, + "baseline_samples": 14, + "load_samples": 24 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "baseline_bytes": 71565312, + "sampled_peak_bytes": 71722598, + "delta_bytes": 157286, + "baseline_samples": 14, + "load_samples": 24 + } + }, + "memory_samples": [ + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 10.56156, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71523369, + "limit_bytes": 2147483648, + "elapsed_s": 10.561597, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 10.561611, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71523369, + "limit_bytes": 2147483648, + "elapsed_s": 10.561622, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 11.062442, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71523369, + "limit_bytes": 2147483648, + "elapsed_s": 11.062517, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 11.062548, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71523369, + "limit_bytes": 2147483648, + "elapsed_s": 11.062574, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 11.561382, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71533855, + "limit_bytes": 2147483648, + "elapsed_s": 11.561413, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 11.561426, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71533855, + "limit_bytes": 2147483648, + "elapsed_s": 11.561438, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 12.061979, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71533855, + "limit_bytes": 2147483648, + "elapsed_s": 12.062034, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 12.062061, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71533855, + "limit_bytes": 2147483648, + "elapsed_s": 12.062087, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68702700, + "limit_bytes": 2147483648, + "elapsed_s": 12.561678, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71544340, + "limit_bytes": 2147483648, + "elapsed_s": 12.561714, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68702700, + "limit_bytes": 2147483648, + "elapsed_s": 12.561731, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71544340, + "limit_bytes": 2147483648, + "elapsed_s": 12.561744, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68702700, + "limit_bytes": 2147483648, + "elapsed_s": 13.06181, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71544340, + "limit_bytes": 2147483648, + "elapsed_s": 13.061845, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68702700, + "limit_bytes": 2147483648, + "elapsed_s": 13.061862, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71544340, + "limit_bytes": 2147483648, + "elapsed_s": 13.061877, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 13.56167, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71565312, + "limit_bytes": 2147483648, + "elapsed_s": 13.561698, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 13.561711, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71565312, + "limit_bytes": 2147483648, + "elapsed_s": 13.561722, + "case": "limit", + "phase": "baseline" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 14.062147, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71565312, + "limit_bytes": 2147483648, + "elapsed_s": 14.062213, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68681728, + "limit_bytes": 2147483648, + "elapsed_s": 14.062246, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71565312, + "limit_bytes": 2147483648, + "elapsed_s": 14.062282, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68755128, + "limit_bytes": 2147483648, + "elapsed_s": 14.562441, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71722598, + "limit_bytes": 2147483648, + "elapsed_s": 14.562503, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68755128, + "limit_bytes": 2147483648, + "elapsed_s": 14.562535, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71722598, + "limit_bytes": 2147483648, + "elapsed_s": 14.562562, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68755128, + "limit_bytes": 2147483648, + "elapsed_s": 15.06192, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71722598, + "limit_bytes": 2147483648, + "elapsed_s": 15.06196, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 68755128, + "limit_bytes": 2147483648, + "elapsed_s": 15.062042, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 71722598, + "limit_bytes": 2147483648, + "elapsed_s": 15.062062, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 52261028, + "limit_bytes": 2147483648, + "elapsed_s": 15.561919, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 66060288, + "limit_bytes": 2147483648, + "elapsed_s": 15.561989, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 52261028, + "limit_bytes": 2147483648, + "elapsed_s": 15.562029, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 66060288, + "limit_bytes": 2147483648, + "elapsed_s": 15.562095, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 52261028, + "limit_bytes": 2147483648, + "elapsed_s": 16.061898, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 66060288, + "limit_bytes": 2147483648, + "elapsed_s": 16.06194, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 52261028, + "limit_bytes": 2147483648, + "elapsed_s": 16.06196, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 66060288, + "limit_bytes": 2147483648, + "elapsed_s": 16.061977, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 45654999, + "limit_bytes": 2147483648, + "elapsed_s": 16.562204, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 44816138, + "limit_bytes": 2147483648, + "elapsed_s": 16.562276, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 45654999, + "limit_bytes": 2147483648, + "elapsed_s": 16.562324, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 44816138, + "limit_bytes": 2147483648, + "elapsed_s": 16.562367, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 45654999, + "limit_bytes": 2147483648, + "elapsed_s": 17.061768, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 44816138, + "limit_bytes": 2147483648, + "elapsed_s": 17.061879, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 45654999, + "limit_bytes": 2147483648, + "elapsed_s": 17.061901, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 44816138, + "limit_bytes": 2147483648, + "elapsed_s": 17.061919, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 17.562025, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 17.562064, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 17.562083, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 17.562098, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 18.06163, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 18.061669, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 18.061691, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 18.061722, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 18.561961, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 18.562048, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 18.562099, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 18.562141, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 19.061725, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 19.061787, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 19.061816, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 19.061842, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 19.562191, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 19.562284, + "case": "limit", + "phase": "load" + }, + { + "container_id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "name": "lab5-fix-check-api-1", + "used_bytes": 44134564, + "limit_bytes": 2147483648, + "elapsed_s": 19.562325, + "case": "limit", + "phase": "load" + }, + { + "container_id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "name": "lab5-fix-check-api-2", + "used_bytes": 41250980, + "limit_bytes": 2147483648, + "elapsed_s": 19.562361, + "case": "limit", + "phase": "load" + } + ], + "memory_error": null, + "results": [ + { + "index": 0, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-6030-79d7-a33d-10cc528f596e", + "latency_ms": 3060.216, + "start_offset_ms": 1.907 + }, + { + "index": 1, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-6026-7be8-b7dc-0027a8fb0048", + "latency_ms": 2795.602, + "start_offset_ms": 0.514 + }, + { + "index": 2, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-6019-7ece-80cb-478dcff80bf9", + "latency_ms": 2186.972, + "start_offset_ms": 0.071 + }, + { + "index": 3, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-6023-7954-9eab-c4b347261aef", + "latency_ms": 2890.313, + "start_offset_ms": 0.689 + }, + { + "index": 4, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-6040-74cb-8b87-ebf2473dd5d9", + "latency_ms": 3102.734, + "start_offset_ms": 0.254 + }, + { + "index": 5, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-601d-7c52-91d0-d625127cb680", + "latency_ms": 2480.11, + "start_offset_ms": 0.828 + }, + { + "index": 6, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-6017-74d1-9c55-009a8d707b52", + "latency_ms": 1903.308, + "start_offset_ms": 0.953 + }, + { + "index": 7, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-601e-7ea8-9690-e1de214dba5a", + "latency_ms": 3068.733, + "start_offset_ms": 1.285 + }, + { + "index": 8, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "8286b4e56e3b403f", + "attachment_id": "01a1f3da-6012-7758-bcd5-a5ea94d884f0", + "latency_ms": 1682.943, + "start_offset_ms": 1.436 + }, + { + "index": 9, + "status": 202, + "ok": true, + "request_bytes": 10485760, + "instance_id": "d26ceaab88586013", + "attachment_id": "01a1f3da-600f-7dd0-9178-7af44a0de979", + "latency_ms": 1686.884, + "start_offset_ms": 1.079 + } + ], + "download": { + "ok": true, + "bytes": 10485760, + "sha256": "d4b8267b4133da413b119c57570ec79a46be76e134e06c5a6931a4943dea325e", + "status": 200 + }, + "state_before": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "state_after": { + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46": { + "id": "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85": { + "id": "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85", + "oom_killed": false, + "running": true, + "restart_count": 0, + "exit_code": 0 + } + }, + "restart_or_oom_detected": false, + "ok": true + } + ], + "api_container_ids": [ + "9d162163992ba271217f0da29110cbd716d584ef457b8d8de101421f595c2b46", + "4b4655ef52a9450913a942f43737acf87998b0a334558d5f22654c9156029f85" + ], + "stage": "complete" +} diff --git a/evidence/worker-off-after-fixes.json b/evidence/worker-off-after-fixes.json new file mode 100644 index 00000000..e8d6bc22 --- /dev/null +++ b/evidence/worker-off-after-fixes.json @@ -0,0 +1,42 @@ +{ + "checked_at": "2026-11-19T12:50:55.167618+00:00", + "project": "lab5-fix-check", + "sequence": [ + "stop worker", + "upload", + "send attached and ordinary messages", + "download before ready", + "restart API replicas and broker (retained volumes)", + "read queued metadata and both messages", + "start worker", + "download original and JPEG" + ], + "attachment_id": "01a1f3db-df50-7817-8ecf-6c91fc045fba", + "upload_status": 202, + "linked_message_id": "01a1f3db-e2a8-76b6-8e68-90abca82c3ce", + "text_message_id": "01a1f3db-e2c2-7984-9e83-608c5706ae77", + "before_restart": { + "job_id": "3ed6676b-cb56-422a-9338-c6b08e64dc42", + "state": "queued", + "attempt": 1 + }, + "content_before_ready": 409, + "after_restart": "queued", + "both_messages_in_history": true, + "final_job": { + "job_id": "3ed6676b-cb56-422a-9338-c6b08e64dc42", + "state": "ready", + "attempt": 1 + }, + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail_sha256": "d7cc662e82b52806da6245e82ad3c0c699ef7b4c6107cb2ee451e05930d3eabb", + "jpeg_dimensions": [ + 256, + 160 + ], + "variants": [ + "crop", + "optimized", + "thumbnail" + ] +} diff --git a/evidence/worker-off-restart.json b/evidence/worker-off-restart.json new file mode 100644 index 00000000..6616093c --- /dev/null +++ b/evidence/worker-off-restart.json @@ -0,0 +1,42 @@ +{ + "checked_at": "2026-11-19T08:15:51.864246+00:00", + "project": "lab5-review", + "sequence": [ + "stop worker", + "upload", + "send attached and ordinary messages", + "download before ready", + "restart API replicas and broker (retained volumes)", + "read queued metadata and both messages", + "start worker", + "download original and JPEG" + ], + "attachment_id": "01a1f2df-496d-79e3-8209-e53c5f5c96e1", + "upload_status": 202, + "linked_message_id": "01a1f2df-4d26-75ed-9ff3-9dcfbf2b43ed", + "text_message_id": "01a1f2df-4d5e-74f8-8acf-72f37d38f663", + "before_restart": { + "job_id": "a1077353-35f7-4b1f-876f-22b9391df6d6", + "state": "queued", + "attempt": 1 + }, + "content_before_ready": 409, + "after_restart": "queued", + "both_messages_in_history": true, + "final_job": { + "job_id": "a1077353-35f7-4b1f-876f-22b9391df6d6", + "state": "ready", + "attempt": 1 + }, + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail_sha256": "d7cc662e82b52806da6245e82ad3c0c699ef7b4c6107cb2ee451e05930d3eabb", + "jpeg_dimensions": [ + 256, + 160 + ], + "variants": [ + "crop", + "optimized", + "thumbnail" + ] +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..46837d52 --- /dev/null +++ b/go.mod @@ -0,0 +1,119 @@ +module messenger + +go 1.27.1 + +require ( + github.com/docker/docker v28.3.3+incompatible + github.com/docker/go-connections v0.6.0 + github.com/gin-gonic/gin v1.12.0 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 + github.com/pressly/goose/v3 v3.27.0 + github.com/rabbitmq/amqp091-go v1.10.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/testcontainers/testcontainers-go v0.39.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 + github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 + google.golang.org/grpc v1.84.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/creack/pty v1.1.24 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mdelapenya/tlscert v0.2.0 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.4.3 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.10.2 // indirect + github.com/stretchr/testify v1.12.1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..96bcfa2d --- /dev/null +++ b/go.sum @@ -0,0 +1,330 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.27.0 h1:/D30gVTuQhu0WsNZYbJi4DMOsx1lNq+6SkLe+Wp59BM= +github.com/pressly/goose/v3 v3.27.0/go.mod h1:3ZBeCXqzkgIRvrEMDkYh1guvtoJTU5oMMuDdkutoM78= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= +github.com/testcontainers/testcontainers-go v0.39.0/go.mod h1:qmHpkG7H5uPf/EvOORKvS6EuDkBUPE3zpVGaH9NL7f8= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= +github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0 h1:b5BwugyCMHrFfZYCSFKY9IHMhoWtR+REcN2ddcCQv6g= +github.com/testcontainers/testcontainers-go/modules/valkey v0.39.0/go.mod h1:lgNiE/W5RPjOu1S8fZ9Z9e63JU8wyaxNQ43Umv6kroQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/valkey-io/valkey-go v1.0.59 h1:W67Z0UY+Qqk3k8NKkFCFlM3X4yQUniixl7dSJAch2Qo= +github.com/valkey-io/valkey-go v1.0.59/go.mod h1:bHmwjIEOrGq/ubOJfh5uMRs7Xj6mV3mQ/ZXUbmqpjqY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679 h1:KmqdJU4vrNcxy/6qdg3JduZtalEXrJLspVltnR1cE+8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260918162117-cecb64721679/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ= +modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..50795f51 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,76 @@ +# The single entry point: round-robin over the API replicas found through +# Docker DNS, with active readiness checks and no client stickiness. + +global + log stdout format raw local0 info + maxconn 20000 + # The container has at most one CPU; by default HAProxy starts a thread + # per host CPU and fights the quota. + nbthread 1 + +defaults + mode http + log global + option dontlognull + timeout connect 1s + timeout client 30s + # Above the API's 5 s request deadline: the API answers a slow request + # itself, this only cuts off a hung replica. + timeout server 7s + timeout http-request 10s + timeout http-keep-alive 10s + timeout check 2s + # Below the 500 ms latency objective: excess load gets a fast 503. + timeout queue 300ms + # Re-sent only when nothing reached the API: a timeout or an empty + # response may follow a committed write. + retries 2 + option redispatch 1 + retry-on conn-failure + # Errors produced by the balancer itself use the contract's envelope. + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + +# `api` resolves to every running replica, re-resolved every second; an +# address that disappears leaves at once. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + hold obsolete 0s + accepted_payload_size 8192 + +frontend entry + bind :8080 + option httplog + unique-id-format %[uuid(4),regsub(-,,g)] + # Replaces any client value; the API keeps it, so one id spans both logs. + http-request set-header X-Request-Id %[unique-id] + capture response header X-Instance-Id len 32 + capture response header X-Request-Id len 40 + http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 } + default_backend replicas + +backend replicas + # Binary handlers have a 30s budget, ordinary API handlers retain 5s. + http-request set-timeout server 35s if { path_reg ^/api/v1/chats/[^/]+/attachments$|^/api/v1/attachments/[^/]+/content$ } + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. + # maxconn 20: the replica's 10 pooled database connections plus requests + # that need no database; the rest waits in the queue. + default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 + # Up to five replicas, doubled while docker rollout replaces them. + server-template api 10 api:8080 + +listen stats + bind :8404 + no log + stats enable + stats uri / + stats refresh 2s diff --git a/infra/s3/README.md b/infra/s3/README.md new file mode 100644 index 00000000..cc9c0bc5 --- /dev/null +++ b/infra/s3/README.md @@ -0,0 +1,77 @@ +# Private object storage + +`s3` runs SeaweedFS 4.53 in one process. The image is pinned by digest in +`compose.yaml` and `internal/testenv/s3.go`. `seaweedfs.options` is shared by +the stand and the disposable integration-test server. + +S3 HTTP (8333) and SeaweedFS's companion gRPC (18333) listen on the private +container network. This image has no flag to disable the companion listener; +its identity-cache mutations reject remote peers, which the integration test +checks. Master, volume and filer HTTP/gRPC endpoints bind to loopback. +No host port is published. `/data` +contains both object volumes and the image's LevelDB filer metadata; the +named `s3data` volume must be retained together with PostgreSQL data. + +Run `sh scripts/init-secrets.sh` before starting the stand. It preserves the +existing PostgreSQL and Valkey credentials and creates three independent S3 +identities plus the static server configuration. Credential values are never +printed. Repeated initialization keeps all existing keys. Set `S3_BUCKET` in +the environment or the local `.env` before the first initialization; changing +it later requires a planned migration of identities and retained data. + +The server loads its static identity secret before serving requests. There is +no anonymous identity. The bootstrap identity has administrative rights and +its individual key files are mounted only in `s3-init`. The API identity has +`Read:` and `Write:`; it cannot list objects or administer +buckets. The reconciler identity adds `List:`. All three identities +are necessarily present in the S3 server's own identity configuration secret. + +`s3-init` creates the bucket idempotently and waits for storage before API +replicas start. An anonymous 403 and bucket metadata alone do not prove that +retained volume locations have registered with the elected master. Bootstrap +first reads a persistent `.health/storage-v1` marker (creating it only when +absent), then verifies a fresh write/read/delete. This startup-only barrier +uses the caller's deadline, capped at 60 seconds; ordinary S3 operations and +API readiness do not retry through it. The marker is outside the reconciler's +`staging/` and `blobs/` namespaces. `scripts/restart.sh` already runs this +bootstrap before restoring API replicas. + +The bootstrap uses `/usr/local/bin/s3init`. The optional `recon` profile runs +`/usr/local/bin/reconcile`, with reconciler credentials mapped to the standard +`S3_ACCESS_KEY_ID_FILE` and `S3_SECRET_ACCESS_KEY_FILE` paths: + +```sh +docker compose --profile recon run --rm reconcile +docker compose --profile recon run --rm reconcile --apply +``` + +The API receives its own standard S3 key files. Its background reconciliation +worker must construct a separate store from +`S3_RECONCILE_ACCESS_KEY_ID_FILE` and `S3_RECONCILE_SECRET_ACCESS_KEY_FILE`; +listing with the API identity is denied. S3 is intentionally absent from API +readiness so text traffic can remain available during an object-store outage. + +The S3 cgroup is limited to one CPU, 512 MiB, and 256 processes. Upload limits +are four concurrent requests and 64 MiB in flight; reader buffers have a +32 MiB budget and the chunk cache is disabled. Disk allocation is capped at +64 volumes of approximately 128 MiB each (with normal volume rollover +overhead); this is a lab capacity limit, not a filesystem quota. + +Each API replica has 2 GiB and `GOMEMLIMIT=1536MiB`, with one image decoder. +Pixels alone are not the full budget: progressive JPEG retains coefficients, +and a wide RGBA64 PNG keeps two large scanlines beside its pixel buffer. +The 40 MP RGB/CMYK progressive JPEG and wide-PNG measurements are recorded in +`evidence/decoder-memory-*.json`; this does not change the S3 service's 512 MiB limit. + +Verify the adapter and real server with Docker running: + +```sh +go test -race -count=1 ./internal/platform/s3store +``` + +`testenv.StartS3(ctx)` returns `Store` with reconciler rights, `APIStore`, +`Admin`, endpoint and credentials. `Recreate(ctx)` replaces only its own +container while keeping its named volume and does not recreate the bucket. +It waits for the same retained-marker and write/read/delete barrier before +returning. `Start(ctx)` does so as well. Always call `Close()` to remove the +fixture's container and volume. diff --git a/infra/s3/seaweedfs.options b/infra/s3/seaweedfs.options new file mode 100644 index 00000000..db8c1c1f --- /dev/null +++ b/infra/s3/seaweedfs.options @@ -0,0 +1,31 @@ +dir=/data +ip=127.0.0.1 +ip.bind=127.0.0.1 +master.telemetry=false +master.volumeSizeLimitMB=128 +volume.port=9340 +volume.max=64 +volume.index=leveldb +volume.fileSizeLimitMB=10 +volume.concurrentUploadLimitMB=64 +volume.concurrentDownloadLimitMB=64 +volume.readBufferSizeMB=1 +volume.preStopSeconds=1 +filer=true +filer.exposeDirectoryData=false +filer.disableDirListing=true +filer.concurrentFileUploadLimit=4 +filer.concurrentUploadLimitMB=64 +s3=true +s3.ip.bind=0.0.0.0 +s3.port=8333 +s3.config=/run/secrets/s3_identities +s3.autoCreateBucket=false +s3.iam=false +s3.port.iceberg=0 +s3.port.lance=0 +s3.concurrentFileUploadLimit=4 +s3.concurrentUploadLimitMB=64 +s3.readerCacheSizeMB=32 +s3.cacheCapacityMB=0 +s3.idleTimeout=35 diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..d7e96bff --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,19 @@ +# Valkey is the only store of user sessions, so it is their source of truth. +# Every write is fsynced before the reply: an acknowledged login survives a +# crash or a restart. Only logins and logouts write; session lookups are reads +# and pay nothing for this. +appendonly yes +appendfsync always +dir /data + +# The append-only file already persists everything; RDB snapshots are off. +save "" + +# Sessions are never evicted to free memory: once maxmemory is reached the +# store refuses new sessions (login answers 503) instead of silently logging +# users out. A session key takes about a hundred bytes. +maxmemory 256mb +maxmemory-policy noeviction + +# requirepass comes from the compose secret, never from the repository. +include /run/secrets/valkey_auth diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 00000000..9ab9a811 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,99 @@ +// Package app is the composition root: it builds every module with its +// stores and mounts them on one HTTP engine. Both the server binary and the +// integration tests start from here. +package app + +import ( + "context" + "log/slog" + "time" + + "github.com/gin-gonic/gin" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "messenger/internal/attachment" + attachmentstore "messenger/internal/attachment/store" + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/chat" + chatstore "messenger/internal/chat/store" + "messenger/internal/health" + "messenger/internal/platform/httpx" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +type App struct { + Engine *gin.Engine + Health *health.Handler + // Chats is exposed for the background cleanup of idempotency keys. + Chats *chat.Service +} + +// Deps are the shared resources the modules are built on; their lifetime is +// owned by the caller. +type Deps struct { + Log *slog.Logger + DB *pgxpool.Pool + Sessions *redis.Client + SessionTTL time.Duration + MaxBodyBytes int64 + // Zero timeouts use the defaults below. + RequestTimeout time.Duration + BinaryRequestTimeout time.Duration + InstanceID string + // IdempotencyTTL zero means the contract's 24 hours. + IdempotencyTTL time.Duration + Blobs attachment.Blobs + AttachmentsCfg attachment.Config +} + +// defaultRequestTimeout stays below the server's write timeout, so a +// request that gives up on a dependency can still write its 503. +const defaultRequestTimeout = 5 * time.Second + +// Binary transfers and decoding share this budget, below the 35s socket timeout. +const defaultBinaryRequestTimeout = 30 * time.Second + +const defaultIdempotencyTTL = 24 * time.Hour + +func New(d Deps) *App { + users := user.NewService(userstore.NewUsers(d.DB)) + keyTTL := d.IdempotencyTTL + if keyTTL == 0 { + keyTTL = defaultIdempotencyTTL + } + chatRepository := chatstore.NewChats(d.DB) + chats := chat.NewService(chatRepository, chatstore.NewMessages(d.DB), users, keyTTL) + + engine := httpx.NewEngine(d.Log, d.MaxBodyBytes, d.InstanceID) + probes := health.New( + health.Check{Name: "postgres", Probe: d.DB.Ping}, + health.Check{Name: "valkey", Probe: func(ctx context.Context) error { return d.Sessions.Ping(ctx).Err() }}, + ) + probes.Register(engine) + + sessions := auth.NewService(users, authstore.NewSessions(d.Sessions), d.SessionTTL) + + timeout := d.RequestTimeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + public := engine.Group("/api/v1", httpx.Timeout(timeout)) + protected := public.Group("", auth.RequireSession(sessions)) + auth.NewHandler(sessions).Register(public, protected) + user.NewHandler(users).Register(public, protected) + chat.NewHandler(chats).Register(protected) + attachments := attachment.NewService( + attachmentstore.NewAttachments(d.DB), chatRepository, d.Blobs, d.AttachmentsCfg, + ) + binaryTimeout := d.BinaryRequestTimeout + if binaryTimeout == 0 { + binaryTimeout = defaultBinaryRequestTimeout + } + binaryProtected := engine.Group("/api/v1", httpx.Timeout(binaryTimeout), auth.RequireSession(sessions)) + attachment.NewHandler(attachments).Register(protected, binaryProtected) + + return &App{Engine: engine, Health: probes, Chats: chats} +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 00000000..a9741ecc --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,523 @@ +package app_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/app" + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/testenv" + "messenger/internal/user" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +const testPassword = "correct-horse-battery" + +const testInstance = "00000000000000a1" + +// errorBody mirrors the contract error envelope. +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type response struct { + status int + requestID string + challenge string + raw []byte +} + +// tester drives the assembled application in-process. +type tester struct { + t *testing.T + a *app.App +} + +// account is a registered user with a live session. +type account struct { + user.Response + token string +} + +// sessionBody mirrors the contract Session schema. +type sessionBody struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +func newTester(t *testing.T) *tester { + t.Helper() + return newTesterWithTTL(t, time.Hour) +} + +// newTesterWithTTL assembles the application over empty stores. +func newTesterWithTTL(t *testing.T, ttl time.Duration) *tester { + t.Helper() + db.Reset(t) + kv.Reset(t) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + return &tester{t: t, a: app.New(app.Deps{ + Log: log, + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: ttl, + MaxBodyBytes: 64 * 1024, + InstanceID: testInstance, + })} +} + +// do sends one request, authenticated with the bearer token when it is set. +// A string body is sent verbatim, anything else is JSON-encoded; both get a +// JSON content type. Every response is checked for X-Request-Id, error +// bodies for the contract envelope, and every 401 for the challenge of the +// scheme the route expects. +func (ts *tester) do(method, path, token string, body any, headers map[string]string) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, encode(ts.t, body)) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + + res := response{ + status: rec.Code, + requestID: rec.Header().Get("X-Request-Id"), + challenge: rec.Header().Get("WWW-Authenticate"), + raw: rec.Body.Bytes(), + } + if res.requestID == "" { + ts.t.Fatalf("%s %s: missing X-Request-Id", method, path) + } + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + ts.t.Fatalf("%s %s: X-Instance-Id = %q, want %q", method, path, got, testInstance) + } + if res.status >= 400 { + var eb errorBody + if err := json.Unmarshal(res.raw, &eb); err != nil { + ts.t.Fatalf("%s %s: error body is not the contract envelope: %v: %s", method, path, err, res.raw) + } + if eb.Error.RequestID != res.requestID || eb.Error.Code == "" || eb.Error.Message == "" { + ts.t.Fatalf("%s %s: bad error envelope: %s", method, path, res.raw) + } + } + if res.status == http.StatusUnauthorized { + scheme := "Bearer " + if path == "/api/v1/auth/sessions" { + scheme = "Basic " + } + if !strings.HasPrefix(res.challenge, scheme) { + ts.t.Fatalf("%s %s: 401 challenge = %q, want scheme %q", method, path, res.challenge, scheme) + } + } + return res +} + +func encode(t *testing.T, body any) io.Reader { + t.Helper() + switch b := body.(type) { + case nil: + return nil + case string: + return strings.NewReader(b) + default: + raw, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + return bytes.NewReader(raw) + } +} + +// expect asserts the status and, for errors, the contract code. +func (ts *tester) expect(res response, status int, code string) { + ts.t.Helper() + if res.status != status { + ts.t.Fatalf("status = %d, want %d; body %s", res.status, status, res.raw) + } + if code != "" { + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + if eb.Error.Code != code { + ts.t.Fatalf("code = %q, want %q; body %s", eb.Error.Code, code, res.raw) + } + } +} + +// decode parses a success body into a typed DTO and rejects unknown fields, +// which enforces the additionalProperties=false rule of the contract. +func decode[T any](t *testing.T, res response) T { + t.Helper() + var out T + dec := json.NewDecoder(bytes.NewReader(res.raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&out); err != nil { + t.Fatalf("body does not match %T: %v: %s", out, err, res.raw) + } + return out +} + +// user registers a user and logs it in. +func (ts *tester) user(username, display string) account { + ts.t.Helper() + body := map[string]string{"username": username, "display_name": display, "password": testPassword} + res := ts.do(http.MethodPost, "/api/v1/users", "", body, nil) + ts.expect(res, http.StatusCreated, "") + registered := decode[user.Response](ts.t, res) + return account{Response: registered, token: ts.login(username, testPassword).Token} +} + +func basic(username, password string) map[string]string { + raw := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + return map[string]string{"Authorization": "Basic " + raw} +} + +func (ts *tester) login(username, password string) sessionBody { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic(username, password)) + ts.expect(res, http.StatusCreated, "") + return decode[sessionBody](ts.t, res) +} + +func (ts *tester) chat(token, title string, members ...string) chat.ChatResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats", token, map[string]any{"title": title, "member_ids": members}, nil) + ts.expect(res, http.StatusCreated, "") + return decode[chat.ChatResponse](ts.t, res) +} + +func (ts *tester) message(token, chatID, text string) chat.MessageResponse { + ts.t.Helper() + res := ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, map[string]string{"text": text}, freshKey()) + ts.expect(res, http.StatusCreated, "") + return decode[chat.MessageResponse](ts.t, res) +} + +func freshKey() map[string]string { + return map[string]string{"Idempotency-Key": uuid.NewString()} +} + +func (ts *tester) chats(token, query string) pagination.PageResponse[chat.ChatResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.ChatResponse]](ts.t, res) +} + +func (ts *tester) history(token, chatID, query string) pagination.PageResponse[chat.MessageResponse] { + ts.t.Helper() + res := ts.do(http.MethodGet, "/api/v1/chats/"+chatID+"/messages"+query, token, nil, nil) + ts.expect(res, http.StatusOK, "") + return decode[pagination.PageResponse[chat.MessageResponse]](ts.t, res) +} + +func cursorQuery(limit int, cursor *string) string { + q := fmt.Sprintf("?limit=%d", limit) + if cursor != nil { + q += "&cursor=" + url.QueryEscape(*cursor) + } + return q +} + +func TestHealthEndpoints(t *testing.T) { + ts := newTester(t) + for _, path := range []string{"/health/live", "/health/ready"} { + res := ts.do(http.MethodGet, path, "", nil, nil) + ts.expect(res, http.StatusOK, "") + if strings.TrimSpace(string(res.raw)) != `{"status":"ok"}` { + t.Fatalf("%s body = %s", path, res.raw) + } + } + ts.a.Health.SetReady(false) + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") +} + +func TestRegisterUserShapeAndConflicts(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + if alice.ID == "" || alice.Username != "alice" || alice.DisplayName != "Алиса" { + t.Fatalf("user = %+v", alice) + } + if !strings.HasSuffix(alice.CreatedAt, "Z") || len(alice.CreatedAt) != len("2026-09-01T12:00:00.000Z") { + t.Fatalf("created_at = %q", alice.CreatedAt) + } + post := func(body any) response { return ts.do(http.MethodPost, "/api/v1/users", "", body, nil) } + ts.expect(post(map[string]string{"username": "alice", "display_name": "Other", "password": testPassword}), http.StatusConflict, "conflict") + ts.expect(post(map[string]string{"username": "bob", "display_name": "Bob"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "Alice", "display_name": "x"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]string{"username": "bob", "display_name": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": "bob", "display_name": "x", "password": "no"}), http.StatusBadRequest, "invalid_request") + ts.expect(post(`{"username":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(map[string]any{"username": 5, "display_name": "x"}), http.StatusBadRequest, "invalid_request") +} + +func TestCurrentUserIsTheTokenOwner(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + ts.user("bob", "Боб") + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got != alice.Response { + t.Fatalf("me = %+v, want %+v", got, alice.Response) + } +} + +func TestChatCreationRules(t *testing.T) { + ts := newTester(t) + alice, bob := ts.user("alice", "A"), ts.user("bob", "B") + created := ts.chat(alice.token, "Проект по ВНП", alice.ID, bob.ID) + if created.Title != "Проект по ВНП" || len(created.MemberIDs) != 2 || created.MemberIDs[0] != alice.ID || created.MemberIDs[1] != bob.ID { + t.Fatalf("chat = %+v", created) + } + unknown := "22222222-2222-4222-8222-222222222222" + invalid := map[string]any{ + "creator missing": map[string]any{"title": "t", "member_ids": []string{bob.ID}}, + "duplicate": map[string]any{"title": "t", "member_ids": []string{alice.ID, alice.ID}}, + "unknown member": map[string]any{"title": "t", "member_ids": []string{alice.ID, unknown}}, + "malformed uuid": map[string]any{"title": "t", "member_ids": []string{alice.ID, "not-a-uuid"}}, + "empty members": map[string]any{"title": "t", "member_ids": []string{}}, + "blank title": map[string]any{"title": "", "member_ids": []string{alice.ID}}, + "long title": map[string]any{"title": strings.Repeat("я", 101), "member_ids": []string{alice.ID}}, + "missing members": map[string]any{"title": "t"}, + "extra field": map[string]any{"title": "t", "member_ids": []string{alice.ID}, "owner": alice.ID}, + } + for name, body := range invalid { + res := ts.do(http.MethodPost, "/api/v1/chats", alice.token, body, nil) + if res.status != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body %s", name, res.status, res.raw) + } + } + ts.expect(ts.do(http.MethodPost, "/api/v1/chats", "", map[string]any{"title": "t", "member_ids": []string{alice.ID}}, nil), http.StatusUnauthorized, "unauthorized") +} + +func TestChatVisibilityAndListing(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + created := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + created.ID + + res := ts.do(http.MethodGet, path, bob.token, nil, nil) + ts.expect(res, http.StatusOK, "") + if got := decode[chat.ChatResponse](t, res); got.ID != created.ID || len(got.MemberIDs) != 2 { + t.Fatalf("GET chat = %+v", got) + } + ts.expect(ts.do(http.MethodGet, path, eve.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/33333333-3333-4333-8333-333333333333", alice.token, nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/not-a-uuid", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, path, "", nil, nil), http.StatusUnauthorized, "unauthorized") + + listed := ts.chats(bob.token, "?limit=100") + if len(listed.Items) != 1 || listed.Items[0].ID != created.ID || listed.NextCursor != nil { + t.Fatalf("bob list = %+v", listed) + } + hidden := ts.do(http.MethodGet, "/api/v1/chats?limit=100", eve.token, nil, nil) + ts.expect(hidden, http.StatusOK, "") + if !strings.Contains(string(hidden.raw), `"items":[]`) || !strings.Contains(string(hidden.raw), `"next_cursor":null`) { + t.Fatalf("empty list must serialize items as [] and next_cursor as null: %s", hidden.raw) + } +} + +func TestChatListPagination(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + var expected []string + for i := 0; i < 5; i++ { + expected = append(expected, ts.chat(alice.token, fmt.Sprintf("chat %d", i), alice.ID).ID) + } + var walked []string + var cursor *string + for pages := 0; pages < 10; pages++ { + page := ts.chats(alice.token, cursorQuery(2, cursor)) + for _, c := range page.Items { + walked = append(walked, c.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walked %v, want %v", walked, expected) + } + + bob := ts.user("bob", "B") + foreign := ts.chats(alice.token, "?limit=1").NextCursor + ts.expect(ts.do(http.MethodGet, "/api/v1/chats"+cursorQuery(50, foreign), bob.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?cursor=", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats?limit=abc", alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessagesEndToEnd(t *testing.T) { + ts := newTester(t) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "Контрактный тест", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + + first := ts.message(alice.token, room.ID, "Привет, мир 👋") + second := ts.message(bob.token, room.ID, "Сообщение Боба") + if first.SenderID != alice.ID || second.SenderID != bob.ID { + t.Fatalf("sender must come from the bearer token: %+v %+v", first, second) + } + if first.ChatID != room.ID || first.Text != "Привет, мир 👋" { + t.Fatalf("message = %+v", first) + } + + page1 := ts.history(bob.token, room.ID, "?limit=1") + if len(page1.Items) != 1 || page1.NextCursor == nil || !reflect.DeepEqual(page1.Items[0], first) { + t.Fatalf("page1 = %+v", page1) + } + page2 := ts.history(alice.token, room.ID, cursorQuery(1, page1.NextCursor)) + if len(page2.Items) != 1 || page2.NextCursor != nil || !reflect.DeepEqual(page2.Items[0], second) { + t.Fatalf("page2 = %+v", page2) + } + + post := func(actor string, body any) response { return ts.do(http.MethodPost, path, actor, body, freshKey()) } + ts.expect(post(alice.token, map[string]string{"text": ""}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": " "}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4001)}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, map[string]string{"text": "x", "sender_id": bob.ID}), http.StatusBadRequest, "invalid_request") + ts.expect(post(alice.token, `{"text":`), http.StatusBadRequest, "invalid_request") + ts.expect(post(eve.token, map[string]string{"text": "чужое"}), http.StatusNotFound, "not_found") + ts.expect(post(eve.token, map[string]string{"text": ""}), http.StatusNotFound, "not_found") + ts.expect(post("", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") + ts.expect(post(alice.token, map[string]string{"text": strings.Repeat("я", 4000)}), http.StatusCreated, "") + + get := func(actor, query string) response { return ts.do(http.MethodGet, path+query, actor, nil, nil) } + ts.expect(get(eve.token, ""), http.StatusNotFound, "not_found") + ts.expect(get(alice.token, "?limit=0"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?limit=101"), http.StatusBadRequest, "invalid_request") + ts.expect(get(alice.token, "?cursor=garbage"), http.StatusBadRequest, "invalid_request") + + other := ts.chat(alice.token, "другой", alice.ID) + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+other.ID+"/messages"+cursorQuery(50, page1.NextCursor), alice.token, nil, nil), http.StatusBadRequest, "invalid_request") +} + +func TestMessageHistoryWalkAcrossPages(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "t", alice.ID) + var expected []string + for i := 0; i < 23; i++ { + expected = append(expected, ts.message(alice.token, room.ID, fmt.Sprintf("m%d", i)).ID) + } + var walked []string + var cursor *string + for { + page := ts.history(alice.token, room.ID, cursorQuery(5, cursor)) + for _, m := range page.Items { + walked = append(walked, m.ID) + } + if page.NextCursor == nil { + break + } + cursor = page.NextCursor + } + if strings.Join(walked, ",") != strings.Join(expected, ",") { + t.Fatalf("walk mismatch:\n%v\n%v", walked, expected) + } +} + +func TestConcurrentSendsProduceExactlyUniqueMessages(t *testing.T) { + ts := newTester(t) + const clients, perClient = 10, 10 + clientsList := make([]account, 0, clients) + members := make([]string, 0, clients) + for i := 0; i < clients; i++ { + c := ts.user(fmt.Sprintf("client_%d", i), "C") + clientsList = append(clientsList, c) + members = append(members, c.ID) + } + room := ts.chat(clientsList[0].token, "нагрузка", members...) + + var wg sync.WaitGroup + for n, c := range clientsList { + wg.Go(func() { + for i := 0; i < perClient; i++ { + res := ts.do(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", c.token, map[string]string{"text": fmt.Sprintf("c%d-m%d", n, i)}, freshKey()) + if res.status != http.StatusCreated { + t.Errorf("client %d: status %d body %s", n, res.status, res.raw) + } + } + }) + } + wg.Wait() + + page := ts.history(clientsList[0].token, room.ID, "?limit=100") + if len(page.Items) != clients*perClient || page.NextCursor != nil { + t.Fatalf("got %d messages, next=%v; want %d and null", len(page.Items), page.NextCursor, clients*perClient) + } + ids := make([]string, 0, len(page.Items)) + unique := make(map[string]struct{}, len(page.Items)) + for _, m := range page.Items { + ids = append(ids, m.ID) + unique[m.ID] = struct{}{} + } + if !sort.StringsAreSorted(ids) { + t.Fatal("history is not ordered by id") + } + if len(unique) != clients*perClient { + t.Fatalf("%d unique ids, want %d", len(unique), clients*perClient) + } +} + +func TestUnknownRouteAndMethodAreContractErrors(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodGet, "/api/v1/nothing", "", nil, nil), http.StatusNotFound, "not_found") + ts.expect(ts.do(http.MethodDelete, "/api/v1/users", "", nil, nil), http.StatusMethodNotAllowed, "invalid_request") + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", "{}", map[string]string{"Content-Type": "text/plain"}), http.StatusUnsupportedMediaType, "unsupported_media_type") + big := map[string]string{"username": "big", "display_name": strings.Repeat("x", 70*1024)} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", big, nil), http.StatusRequestEntityTooLarge, "payload_too_large") +} diff --git a/internal/app/attachment_test.go b/internal/app/attachment_test.go new file mode 100644 index 00000000..13aff429 --- /dev/null +++ b/internal/app/attachment_test.go @@ -0,0 +1,263 @@ +package app_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "image" + "image/jpeg" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/app" + "messenger/internal/attachment" + "messenger/internal/chat" + "messenger/internal/pipeline" + "messenger/internal/testenv" +) + +func TestAttachmentUploadAuthenticatesBeforeParsing(t *testing.T) { + ts := newTester(t) + ts.expect(ts.do(http.MethodPost, "/api/v1/chats/not-a-uuid/attachments?filename=x.png", "", "invalid image", map[string]string{"Content-Type": "image/png"}), http.StatusUnauthorized, "unauthorized") +} + +func TestAttachmentRoutesUseConfiguredDeadlines(t *testing.T) { + for _, binary := range []bool{false, true} { + name := "metadata" + requestTimeout := 40 * time.Millisecond + var binaryTimeout time.Duration + if binary { + name = "upload" + requestTimeout = time.Second + binaryTimeout = 40 * time.Millisecond + } + t.Run(name, func(t *testing.T) { + ts := newTester(t) + actor := ts.user("deadline_user", "User") + ts.a = app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, + SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, + RequestTimeout: requestTimeout, BinaryRequestTimeout: binaryTimeout, + }) + var held []*pgxpool.Conn + for range db.Pool.Config().MaxConns { + conn, err := db.Pool.Acquire(context.Background()) + if err != nil { + t.Fatal(err) + } + held = append(held, conn) + } + defer func() { + for _, conn := range held { + conn.Release() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + path := "/api/v1/attachments/11111111-1111-4111-8111-111111111111" + req := httptest.NewRequest("GET", path, nil).WithContext(ctx) + if binary { + path = "/api/v1/chats/11111111-1111-4111-8111-111111111111/attachments?filename=x.png" + req = httptest.NewRequest("POST", path, strings.NewReader("body")).WithContext(ctx) + req.Header.Set("Content-Type", "image/png") + } + req.Header.Set("Authorization", "Bearer "+actor.token) + rec := httptest.NewRecorder() + started := time.Now() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != 503 || time.Since(started) > 200*time.Millisecond { + t.Fatalf("route exceeded configured budget: status%d elapsed%v body%s", rec.Code, time.Since(started), rec.Body.String()) + } + }) + } +} + +func (ts *tester) binary(method, path, token, mime string, data []byte, length int64) response { + ts.t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(data)) + req.ContentLength = length + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if mime != "" { + req.Header.Set("Content-Type", mime) + } + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + res := response{status: rec.Code, requestID: rec.Header().Get("X-Request-Id"), challenge: rec.Header().Get("WWW-Authenticate"), raw: rec.Body.Bytes()} + if res.requestID == "" || rec.Header().Get("X-Instance-Id") == "" { + ts.t.Fatal("binary response missing correlation headers") + } + return res +} + +func TestAttachmentRealS3RoundTripValidationAndOutage(t *testing.T) { + ctx := context.Background() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + ts := newTester(t) + ts.a = app.New(app.Deps{Log: slog.New(slog.NewTextHandler(io.Discard, nil)), DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 65536, InstanceID: testInstance, Blobs: s3.APIStore}) + alice, bob, eve := ts.user("alice", "A"), ts.user("bob", "B"), ts.user("eve", "E") + room := ts.chat(alice.token, "files", alice.ID, bob.ID) + path := "/api/v1/chats/" + room.ID + "/attachments?filename=" + url.QueryEscape("фото.png") + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + res := ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))) + ts.expect(res, 202, "") + a := decode[attachment.Response](t, res) + digest := sha256.Sum256(data) + if a.SHA256 != hex.EncodeToString(digest[:]) || a.SizeBytes != int64(len(data)) || a.Filename != "фото.png" || a.Status != "queued" || a.ErrorCode != nil || len(a.Variants) != 0 { + t.Fatalf("attachment: %+v", a) + } + base := "/api/v1/attachments/" + a.ID + for _, suffix := range []string{"", "/content"} { + ts.expect(ts.do("GET", base+suffix, eve.token, nil, nil), 404, "not_found") + } + ts.expect(ts.do("GET", base, bob.token, nil, nil), 200, "") + got := ts.binary("GET", base+"/content", bob.token, "", nil, 0) + ts.expect(got, 409, "conflict") + ts.expect(ts.do("GET", base+"/content?variant=thumbnail", bob.token, nil, nil), 409, "conflict") + ts.expect(ts.do("GET", base+"/content?variant=unknown", bob.token, nil, nil), 400, "invalid_request") + msg := ts.do("POST", "/api/v1/chats/"+room.ID+"/messages", bob.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()) + ts.expect(msg, 201, "") + if m := decode[chat.MessageResponse](t, msg); len(m.AttachmentIDs) != 1 || m.AttachmentIDs[0] != a.ID { + t.Fatal("lost attachment on message") + } + other := ts.chat(alice.token, "other", alice.ID) + ts.expect(ts.do("POST", "/api/v1/chats/"+other.ID+"/messages", alice.token, map[string]any{"text": "", "attachment_ids": []string{a.ID}}, freshKey()), 404, "not_found") + for _, tc := range []struct { + name, mime string + data []byte + length int64 + want int + }{ + {"MIME spoof", "image/jpeg", data, int64(len(data)), 415}, + {"broken body", "image/png", data[:len(data)/2], int64(len(data) / 2), 415}, + {"unsupported", "image/gif", data, int64(len(data)), 415}, + {"oversize", "image/png", nil, 10485761, 413}, + {"unknown length", "image/png", data, -1, 400}, + {"empty", "image/png", nil, 0, 400}, + } { + t.Run(tc.name, func(t *testing.T) { + r := ts.binary("POST", path, alice.token, tc.mime, tc.data, tc.length) + if r.status != tc.want { + t.Fatalf("got %d want %d: %s", r.status, tc.want, r.raw) + } + }) + } + for _, name := range []string{"../x.png", `a\b.png`, "bad\nname", strings.Repeat("я", 256)} { + ts.expect(ts.binary("POST", "/api/v1/chats/"+room.ID+"/attachments?filename="+url.QueryEscape(name), alice.token, "image/png", data, int64(len(data))), 400, "invalid_request") + } + var j bytes.Buffer + if err := jpeg.Encode(&j, image.NewRGBA(image.Rect(0, 0, 5, 3)), nil); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, bob.token, "image/jpeg", j.Bytes(), int64(j.Len())), 202, "") + ts.expect(ts.binary("POST", path, eve.token, "image/png", data, int64(len(data))), 404, "not_found") + // An interrupted read is never a 500 or a successfully published identity. + before := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&before); err != nil { + t.Fatal(err) + } + short := ts.binary("POST", path, alice.token, "image/png", data[:10], int64(len(data))) + if short.status != 400 && short.status != 503 { + t.Fatalf("interrupted body: %d %s", short.status, short.raw) + } + after := 0 + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachments`).Scan(&after); err != nil || before != after { + t.Fatalf("interruption published identity: %d -> %d (%v)", before, after, err) + } + // Complete one claimed job with known bytes. Image encoding has its own + // tests; this checks the published version, exact download and ACL on S3. + repo := pipeline.NewStore(db.Pool) + var job pipeline.Job + found, err := repo.PublishOne(ctx, func(_ context.Context, j pipeline.Job) error { job = j; return nil }) + if err != nil || !found || job.AttachmentID.String() != a.ID { + t.Fatalf("outbox: %+v %v", job, err) + } + claim, err := repo.Claim(ctx, job, time.Minute) + if err != nil || claim == nil { + t.Fatal("claim", err) + } + meta := decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil)) + if meta.Status != "processing" || len(meta.Variants) != 0 { + t.Fatal("processing snapshot", meta) + } + if err := s3.Store.Copy(ctx, claim.SourceKey, attachment.BlobKey(claim.Hash)); err != nil { + t.Fatal(err) + } + bodies := map[string][]byte{"original": data} + var outputs []pipeline.Variant + for _, name := range []string{"thumbnail", "crop", "optimized"} { + body := []byte("published-" + name) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + digest := sha256.Sum256(body) + key := pipeline.VariantKey(claim.AttachmentID, claim.PipelineVersion, name, digest[:]) + if err := s3.Store.Put(ctx, key, bytes.NewReader(body), int64(len(body)), mime); err != nil { + t.Fatal(err) + } + outputs = append(outputs, pipeline.Variant{Name: name, Key: key, MIMEType: mime, Width: 1, Height: 1, SizeBytes: int64(len(body)), SHA256: digest[:]}) + bodies[name] = body + } + if ok, err := repo.Complete(ctx, claim, outputs); err != nil || !ok { + t.Fatal("publish", err) + } + if err := repo.Reprocess(ctx, claim.AttachmentID, 2, func(ctx context.Context, key string) (bool, error) { + _, err := s3.Store.Stat(ctx, key) + if errors.Is(err, attachment.ErrObjectMissing) { + return false, nil + } + return err == nil, err + }); err != nil { + t.Fatal(err) + } + meta = decode[attachment.Response](t, ts.do("GET", base, bob.token, nil, nil)) + if meta.Status != "ready" || len(meta.Variants) != 3 || meta.ErrorCode != nil || meta.SHA256 != a.SHA256 { + t.Fatalf("ready snapshot during reprocess: %+v", meta) + } + for _, v := range meta.Variants { + digest := sha256.Sum256(bodies[v.Name]) + if v.SHA256 != hex.EncodeToString(digest[:]) || v.SizeBytes != int64(len(bodies[v.Name])) { + t.Fatal("variant checksum", v) + } + } + for name, body := range bodies { + target := base + "/content?variant=" + name + ts.expect(ts.do("GET", target, eve.token, nil, nil), 404, "not_found") + got := ts.binary("GET", target, bob.token, "", nil, 0) + ts.expect(got, 200, "") + if !bytes.Equal(got.raw, body) { + t.Fatalf("wrong published bytes for %s", name) + } + } + if err := s3.Store.Delete(ctx, outputs[0].Key); err != nil { + t.Fatal(err) + } + ts.expect(ts.do("GET", base+"/content?variant="+outputs[0].Name, bob.token, nil, nil), 404, "not_found") + if err := s3.Stop(ctx); err != nil { + t.Fatal(err) + } + ts.expect(ts.binary("POST", path, alice.token, "image/png", data, int64(len(data))), 503, "unavailable") + ts.expect(ts.do("GET", base+"/content", bob.token, nil, nil), 503, "unavailable") + ts.expect(ts.do("GET", "/health/ready", "", nil, nil), 200, "") + ts.message(alice.token, room.ID, fmt.Sprintf("text works with S3 down %d", time.Now().UnixNano())) +} diff --git a/internal/app/auth_test.go b/internal/app/auth_test.go new file mode 100644 index 00000000..d0d4819b --- /dev/null +++ b/internal/app/auth_test.go @@ -0,0 +1,176 @@ +package app_test + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "messenger/internal/user" +) + +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +func TestLoginReturnsBearerSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "Алиса") + before := time.Now() + + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)) + ts.expect(res, http.StatusCreated, "") + s := decode[sessionBody](t, res) + + if s.TokenType != "Bearer" || len(s.Token) != 43 || s.Token == alice.token { + t.Fatalf("session = %+v; want a fresh 43-character Bearer token", s) + } + if s.User != alice.Response { + t.Fatalf("user = %+v, want %+v", s.User, alice.Response) + } + expires, err := time.Parse(time.RFC3339, s.ExpiresAt) + if err != nil || !strings.HasSuffix(s.ExpiresAt, "Z") { + t.Fatalf("expires_at = %q: %v", s.ExpiresAt, err) + } + if ttl := expires.Sub(before); ttl < 59*time.Minute || ttl > 61*time.Minute { + t.Fatalf("expires_at is %v after login, want the 1h TTL", ttl) + } + if strings.Contains(string(res.raw), "password") || strings.Contains(string(res.raw), "argon2") { + t.Fatalf("session response leaks password data: %s", res.raw) + } +} + +func TestLoginFailuresAreUniformBasicChallenges(t *testing.T) { + ts := newTester(t) + ts.user("alice", "Алиса") + encoded := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) } + cases := map[string]map[string]string{ + "wrong password": basic("alice", testPassword+"!"), + "unknown user": basic("mallory", testPassword), + "no header": nil, + "bearer scheme": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "not base64": {"Authorization": "Basic !!!"}, + "no colon": {"Authorization": "Basic " + encoded("alice")}, + "nul username": basic("a\x00b", testPassword), + "non-utf-8 name": basic("\xff\xfe", testPassword), + } + messages := map[string]string{} + for name, headers := range cases { + res := ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != basicChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, basicChallenge) + } + var eb errorBody + _ = json.Unmarshal(res.raw, &eb) + messages[name] = eb.Error.Message + } + if messages["wrong password"] != messages["unknown user"] { + t.Fatalf("wrong password and unknown user must look the same: %q vs %q", messages["wrong password"], messages["unknown user"]) + } +} + +// The password is UTF-8 and may contain colons; only the first colon of the +// Basic credentials separates the username (contracts/README.md). +func TestLoginAcceptsColonAndUnicodePasswords(t *testing.T) { + ts := newTester(t) + const password = "пароль:с:двоеточием 👋" + body := map[string]string{"username": "alice", "display_name": "Алиса", "password": password} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") + s := ts.login("alice", password) + if s.User.Username != "alice" { + t.Fatalf("logged in as %+v", s.User) + } +} + +func TestProtectedRoutesAcceptOnlyBearer(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + me := func(headers map[string]string) response { + return ts.do(http.MethodGet, "/api/v1/users/me", "", nil, headers) + } + rejected := map[string]map[string]string{ + "no header": nil, + "basic with valid credentials": basic("alice", testPassword), + "x-user-id alone": {"X-User-Id": alice.ID}, + "unknown token": {"Authorization": "Bearer " + strings.Repeat("A", 43)}, + "empty token": {"Authorization": "Bearer "}, + "scheme only": {"Authorization": "Bearer"}, + "other scheme": {"Authorization": "Token " + alice.token}, + "two tokens": {"Authorization": "Bearer " + alice.token + " " + eve.token}, + } + for name, headers := range rejected { + res := me(headers) + ts.expect(res, http.StatusUnauthorized, "unauthorized") + if res.challenge != bearerChallenge { + t.Errorf("%s: WWW-Authenticate = %q, want %q", name, res.challenge, bearerChallenge) + } + } + accepted := map[string]map[string]string{ + "lowercase scheme": {"Authorization": "bearer " + alice.token}, + "several spaces": {"Authorization": "Bearer " + alice.token}, + "x-user-id is ignored": {"Authorization": "Bearer " + alice.token, "X-User-Id": eve.ID}, + } + for name, headers := range accepted { + res := me(headers) + ts.expect(res, http.StatusOK, "") + if got := decode[user.Response](t, res); got.ID != alice.ID { + t.Errorf("%s: caller = %s, want alice", name, got.ID) + } + } +} + +func TestLogoutRevokesOnlyTheCurrentToken(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + other := ts.login("alice", testPassword) + + res := ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil) + ts.expect(res, http.StatusNoContent, "") + if len(res.raw) != 0 { + t.Fatalf("204 must have no body, got %q", res.raw) + } + + for _, again := range []response{ + ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), + ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), + } { + ts.expect(again, http.StatusUnauthorized, "unauthorized") + if again.challenge != bearerChallenge { + t.Fatalf("revoked token: WWW-Authenticate = %q, want %q", again.challenge, bearerChallenge) + } + } + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", other.Token, nil, nil), http.StatusOK, "") +} + +// Valkey enforces the expiry by the Docker VM clock, which may lag this host +// by a fraction of a second, so the 401 is awaited with a tolerance. +func TestSessionExpiresAfterTTL(t *testing.T) { + const ttl, clockTolerance = 2 * time.Second, 3 * time.Second + ts := newTesterWithTTL(t, ttl) + alice := ts.user("alice", "A") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusOK, "") + time.Sleep(ttl) + deadline := time.Now().Add(clockTolerance) + for { + res := ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil) + if res.status == http.StatusUnauthorized { + ts.expect(res, http.StatusUnauthorized, "unauthorized") + return + } + if time.Now().After(deadline) { + t.Fatalf("session still valid %v after its TTL of %v", clockTolerance, ttl) + } + time.Sleep(100 * time.Millisecond) + } +} + +func TestRegistrationIsPublicAndIgnoresAuthorization(t *testing.T) { + ts := newTester(t) + body := map[string]string{"username": "alice", "display_name": "A", "password": testPassword} + headers := map[string]string{"Authorization": "Bearer garbage"} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, headers), http.StatusCreated, "") +} diff --git a/internal/app/failure_test.go b/internal/app/failure_test.go new file mode 100644 index 00000000..155c8000 --- /dev/null +++ b/internal/app/failure_test.go @@ -0,0 +1,101 @@ +package app_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +// newIsolatedTester assembles the application over containers of its own, +// so a test can stop or pause them without disturbing the shared ones. +func newIsolatedTester(t *testing.T, requestTimeout time.Duration) (*tester, *testenv.DB, *testenv.KV) { + t.Helper() + ctx := context.Background() + pg, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pg.Close) + vk, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(vk.Close) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{ + Log: log, + DB: pg.Pool, + Sessions: vk.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + RequestTimeout: requestTimeout, + InstanceID: testInstance, + }) + return &tester{t: t, a: a}, pg, vk +} + +// Without the session store no caller can be authenticated: the API answers +// 503, never 401 (the token is not at fault) and never as an anonymous user. +// Registration does not need the store and keeps working. +func TestValkeyOutageAnswers503(t *testing.T) { + ts, _, vk := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + room := ts.chat(alice.token, "t", alice.ID) + + if err := vk.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/chats/"+room.ID, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/live", "", nil, nil), http.StatusOK, "") + + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusCreated, "") +} + +func TestPostgresOutageAnswers503(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 5*time.Second) + alice := ts.user("alice", "Алиса") + + if err := pg.Stop(context.Background()); err != nil { + t.Fatal(err) + } + + ts.expect(ts.do(http.MethodGet, "/api/v1/chats", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/api/v1/users/me", alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodPost, "/api/v1/auth/sessions", "", nil, basic("alice", testPassword)), http.StatusServiceUnavailable, "unavailable") + body := map[string]string{"username": "bob", "display_name": "Bob", "password": testPassword} + ts.expect(ts.do(http.MethodPost, "/api/v1/users", "", body, nil), http.StatusServiceUnavailable, "unavailable") + ts.expect(ts.do(http.MethodGet, "/health/ready", "", nil, nil), http.StatusServiceUnavailable, "unavailable") +} + +// A database that accepts connections but never answers must not hold the +// request: the request deadline turns it into a 503. +func TestHungPostgresAnswers503WithinTheRequestTimeout(t *testing.T) { + ts, pg, _ := newIsolatedTester(t, 300*time.Millisecond) + alice := ts.user("alice", "Алиса") + + ctx := context.Background() + if err := pg.Pause(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.Unpause(ctx) }) + + for _, path := range []string{"/api/v1/chats", "/health/ready"} { + start := time.Now() + ts.expect(ts.do(http.MethodGet, path, alice.token, nil, nil), http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("%s took %v with a hung database, want it bounded", path, elapsed) + } + } +} diff --git a/internal/app/idempotency_test.go b/internal/app/idempotency_test.go new file mode 100644 index 00000000..e213baab --- /dev/null +++ b/internal/app/idempotency_test.go @@ -0,0 +1,153 @@ +package app_test + +import ( + "bytes" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "messenger/internal/app" +) + +func (ts *tester) send(token, chatID, key string, body any) response { + ts.t.Helper() + return ts.do(http.MethodPost, "/api/v1/chats/"+chatID+"/messages", token, body, map[string]string{"Idempotency-Key": key}) +} + +func TestSendIdempotencyKeyValidation(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "ключи", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + body := map[string]string{"text": "x"} + + ts.expect(ts.do(http.MethodPost, path, alice.token, body, nil), http.StatusBadRequest, "invalid_request") + for name, key := range map[string]string{ + "empty": "", + "too long": strings.Repeat("a", 129), + "inner space": "a b", + "non-ascii": "ключ", + "slash": "a/b", + } { + res := ts.send(alice.token, room.ID, key, body) + if res.status != http.StatusBadRequest { + t.Errorf("%s key: status %d, want 400", name, res.status) + } + } + ts.expect(ts.send(alice.token, room.ID, strings.Repeat("a", 128), body), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "Az09._:-", body), http.StatusCreated, "") + + // Two Idempotency-Key headers are ambiguous: which one names the attempt? + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"x"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Add("Idempotency-Key", "one") + req.Header.Add("Idempotency-Key", "two") + rec := httptest.NewRecorder() + ts.a.Engine.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("two keys: status %d, want 400", rec.Code) + } +} + +func TestReplayReturnsIdenticalBody(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "повторы", alice.ID) + first := ts.send(alice.token, room.ID, "retry-1", map[string]string{"text": "Привет 👋"}) + ts.expect(first, http.StatusCreated, "") + // The same JSON with other spacing is the same request. + again := ts.send(alice.token, room.ID, "retry-1", ` { "text" : "Привет 👋" } `) + ts.expect(again, http.StatusCreated, "") + if !bytes.Equal(first.raw, again.raw) { + t.Fatalf("replay body differs:\n%s\n%s", first.raw, again.raw) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1", len(page.Items)) + } +} + +func TestReplayWithDifferentBodyIs409(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "конфликт", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "первое"}), http.StatusCreated, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "второе"}), http.StatusConflict, "conflict") +} + +// A replay is a new request: a revoked session cannot fetch the stored result. +func TestReplayRechecksSession(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "сессия", alice.ID) + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusCreated, "") + ts.expect(ts.do(http.MethodDelete, "/api/v1/auth/sessions/current", alice.token, nil, nil), http.StatusNoContent, "") + ts.expect(ts.send(alice.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusUnauthorized, "unauthorized") +} + +// The header is checked before access, like the JSON syntax: the answer does +// not depend on the chat, so it reveals nothing about a foreign one. +func TestOutsiderWithoutKeyGets400AndWithKey404(t *testing.T) { + ts := newTester(t) + alice, eve := ts.user("alice", "A"), ts.user("eve", "E") + room := ts.chat(alice.token, "чужой", alice.ID) + path := "/api/v1/chats/" + room.ID + "/messages" + ts.expect(ts.do(http.MethodPost, path, eve.token, map[string]string{"text": "x"}, nil), http.StatusBadRequest, "invalid_request") + ts.expect(ts.send(eve.token, room.ID, "k", map[string]string{"text": "x"}), http.StatusNotFound, "not_found") +} + +// Two applications over the same stores stand for two replicas: retries of one +// key alternate between them and still create one message. +func TestConcurrentReplaysAcrossTwoAppsCreateOneMessage(t *testing.T) { + ts := newTester(t) + alice := ts.user("alice", "A") + room := ts.chat(alice.token, "реплики", alice.ID) + other := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "00000000000000b2", + }) + engines := []http.Handler{ts.a.Engine, other.Engine} + + const attempts = 10 + type result struct { + status int + body string + instance string + } + results := make([]result, attempts) + var wg sync.WaitGroup + for i := range attempts { + wg.Go(func() { + req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room.ID+"/messages", strings.NewReader(`{"text":"один раз"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.token) + req.Header.Set("Idempotency-Key", "shared") + rec := httptest.NewRecorder() + engines[i%2].ServeHTTP(rec, req) + results[i] = result{rec.Code, rec.Body.String(), rec.Header().Get("X-Instance-Id")} + }) + } + wg.Wait() + instances := map[string]bool{} + for i, r := range results { + if r.status != http.StatusCreated || r.body != results[0].body { + t.Fatalf("attempt %d: %d %s; want 201 %s", i, r.status, r.body, results[0].body) + } + instances[r.instance] = true + } + if len(instances) != 2 { + t.Fatalf("instances %v, want both replicas", instances) + } + if page := ts.history(alice.token, room.ID, ""); len(page.Items) != 1 { + t.Fatalf("history has %d messages, want 1: %+v", len(page.Items), page.Items) + } +} diff --git a/internal/attachment/attachment.go b/internal/attachment/attachment.go new file mode 100644 index 00000000..55871fe2 --- /dev/null +++ b/internal/attachment/attachment.go @@ -0,0 +1,45 @@ +package attachment + +import ( + "time" + + "github.com/google/uuid" +) + +type FileStatus string + +const ( + FileQueued FileStatus = "queued" + FileProcessing FileStatus = "processing" + FilePending FileStatus = "pending" + FileReady FileStatus = "ready" + FileFailed FileStatus = "failed" +) + +type Attachment struct { + ID, ChatID, OwnerID uuid.UUID + Filename string + FileHash []byte + SizeBytes int64 + MIMEType string + Status FileStatus + CreatedAt, UpdatedAt time.Time + ErrorCode *string + Variants []Variant +} + +type UploadInput struct { + ChatID uuid.UUID + Filename string + MIMEType string + SizeBytes int64 +} + +type Variant struct { + Name string + ObjectKey string + MIMEType string + SizeBytes int64 + Width, Height int + Hash []byte +} diff --git a/internal/attachment/blob.go b/internal/attachment/blob.go new file mode 100644 index 00000000..3958653b --- /dev/null +++ b/internal/attachment/blob.go @@ -0,0 +1,25 @@ +package attachment + +import ( + "context" + "errors" + "io" + "time" +) + +var ErrObjectMissing = errors.New("object missing") + +type Object struct { + Key string + Size int64 + Modified time.Time +} + +type Blobs interface { + Put(ctx context.Context, key string, body io.Reader, size int64, mimeType string) error + Open(ctx context.Context, key string) (io.ReadCloser, error) + Stat(ctx context.Context, key string) (Object, error) + Copy(ctx context.Context, sourceKey, destinationKey string) error + Delete(ctx context.Context, key string) error + List(ctx context.Context, prefix, after string, limit int) ([]Object, error) +} diff --git a/internal/attachment/handler.go b/internal/attachment/handler.go new file mode 100644 index 00000000..238ee136 --- /dev/null +++ b/internal/attachment/handler.go @@ -0,0 +1,168 @@ +package attachment + +import ( + "encoding/hex" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type Handler struct{ s *Service } + +func NewHandler(s *Service) *Handler { return &Handler{s} } +func (h *Handler) Register(protected, binaryProtected gin.IRouter) { + binaryProtected.POST("/chats/:chat_id/attachments", httpx.WithUser(h.upload)) + protected.GET("/attachments/:attachment_id", httpx.WithUser(h.metadata)) + binaryProtected.GET("/attachments/:attachment_id/content", httpx.WithUser(h.content)) +} + +type Response struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []VariantResponse `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type VariantResponse struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +func toResponse(a Attachment) Response { + variants := make([]VariantResponse, 0, len(a.Variants)) + for _, v := range a.Variants { + variants = append(variants, VariantResponse{ + Name: v.Name, ContentType: v.MIMEType, SizeBytes: v.SizeBytes, + Width: v.Width, Height: v.Height, SHA256: hex.EncodeToString(v.Hash), + }) + } + return Response{ + ID: a.ID.String(), + ChatID: a.ChatID.String(), + OwnerID: a.OwnerID.String(), + Filename: a.Filename, + ContentType: a.MIMEType, + SizeBytes: a.SizeBytes, + SHA256: hex.EncodeToString(a.FileHash), + Status: string(a.Status), + Variants: variants, + ErrorCode: a.ErrorCode, + CreatedAt: httpx.Timestamp(a.CreatedAt), + UpdatedAt: httpx.Timestamp(a.UpdatedAt), + } +} + +func (h *Handler) upload(c *gin.Context, actor uuid.UUID) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + httpx.Error(c, apperr.Invalid("chat_id must be a UUID")) + return + } + names := c.Request.URL.Query()["filename"] + if len(names) != 1 { + httpx.Error(c, apperr.Invalid("one filename is required")) + return + } + if err := ValidateFilename(names[0]); err != nil { + httpx.Error(c, err) + return + } + media, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || len(params) > 0 { + httpx.Error(c, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.s.config.MaxBytes) + input := UploadInput{ + ChatID: id, + Filename: names[0], + MIMEType: media, + SizeBytes: c.Request.ContentLength, + } + a, err := h.s.Upload(c.Request.Context(), actor, input, c.Request.Body) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusAccepted, toResponse(a)) +} +func attachmentID(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("attachment_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("attachment_id must be a UUID") + } + return id, nil +} +func (h *Handler) metadata(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + a, err := h.s.Get(c.Request.Context(), actor, id) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toResponse(a)) +} +func (h *Handler) content(c *gin.Context, actor uuid.UUID) { + id, err := attachmentID(c) + if err != nil { + httpx.Error(c, err) + return + } + variant := "original" + if values, ok := c.Request.URL.Query()["variant"]; ok { + if len(values) != 1 { + httpx.Error(c, apperr.Invalid("one variant is required")) + return + } + variant = values[0] + switch variant { + case "original", "thumbnail", "crop", "optimized": + default: + httpx.Error(c, apperr.Invalid("unknown variant")) + return + } + } + a, r, err := h.s.Content(c.Request.Context(), actor, id, variant) + if err != nil { + httpx.Error(c, err) + return + } + defer func() { + if err := r.Close(); err != nil { + slog.Warn("download reader close failed", "request_id", httpx.RequestID(c), "error", err) + } + }() + c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Type", a.MIMEType) + c.Header("Content-Length", strconv.FormatInt(a.SizeBytes, 10)) + c.Status(http.StatusOK) + if _, err := io.CopyN(c.Writer, r, a.SizeBytes); err != nil { + slog.Error("download interrupted", "request_id", httpx.RequestID(c), "error", err) + } +} diff --git a/internal/attachment/lock.go b/internal/attachment/lock.go new file mode 100644 index 00000000..31b73342 --- /dev/null +++ b/internal/attachment/lock.go @@ -0,0 +1,23 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + + "github.com/jackc/pgx/v5" +) + +// LockHash serializes CAS publication and collection until the transaction ends. +// A collision in the signed first eight digest bytes only adds serialization. +func LockHash(ctx context.Context, tx pgx.Tx, hash []byte) error { + if len(hash) != sha256.Size { + return errors.New("attachment: lock requires a SHA256 hash") + } + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(binary.BigEndian.Uint64(hash[:8]))) + return err +} + +func BlobKey(hash []byte) string { return "blobs/" + hex.EncodeToString(hash) } diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 00000000..0d78dec7 --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,610 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + reconcilePageSize = 500 + reconcileOperationTimeout = 30 * time.Second +) + +type Finding struct { + Kind string `json:"kind"` + Key string `json:"key"` + Action string `json:"action"` + Applied bool `json:"applied"` +} + +// SQLQueries counts SQL statements, including transaction control and locks. +// S3Calls counts calls to the Blobs port, including each list page and deep GET. +type Stats struct { + ElapsedMS int64 `json:"elapsed_ms"` + SQLQueries int64 `json:"sql_queries"` + S3Calls int64 `json:"s3_calls"` + ScannedFiles int64 `json:"scanned_files"` + ScannedObjects int64 `json:"scanned_objects"` + Findings int64 `json:"findings"` + Repaired int64 `json:"repaired"` +} + +type Reconciler struct { + db *pgxpool.Pool + blobs Blobs + grace time.Duration + retention time.Duration +} + +type ReconcileOptions struct { + Apply bool + Deep bool +} + +func NewReconciler(db *pgxpool.Pool, blobs Blobs, grace time.Duration) (*Reconciler, error) { + return NewReconcilerWithRetention(db, blobs, grace, 24*time.Hour) +} + +func NewReconcilerWithRetention(db *pgxpool.Pool, blobs Blobs, grace, retention time.Duration) (*Reconciler, error) { + if retention < time.Second { + return nil, errors.New("reconcile: staging retention must be at least one second") + } + if db == nil || blobs == nil { + return nil, errors.New("reconcile: database and blobs are required") + } + if grace == 0 { + grace = time.Hour + } + if grace < 2*time.Minute { + return nil, errors.New("reconcile: grace must be at least two minutes") + } + return &Reconciler{db: db, blobs: blobs, grace: grace, retention: retention}, nil +} + +// Run streams findings and keeps at most one page of metadata in memory. +// Dry runs only read storage. Each apply candidate is checked again under the +// publisher's transaction lock; no scan snapshot authorizes a later mutation. +func (r *Reconciler) Run(ctx context.Context, options ReconcileOptions, emit func(Finding)) (stats Stats, err error) { + started := time.Now() + defer func() { stats.ElapsedMS = time.Since(started).Milliseconds() }() + run := reconcileRun{ + r: r, + ctx: ctx, + apply: options.Apply, + deep: options.Deep, + emit: emit, + stats: &stats, + cutoff: started.Add(-r.grace), + } + for _, prefix := range []string{"staging/", "variants/", "blobs/"} { + if err = run.objects(prefix); err != nil { + return stats, err + } + } + err = run.files() + return stats, err +} + +type reconcileRun struct { + r *Reconciler + ctx context.Context + apply, deep bool + emit func(Finding) + stats *Stats + cutoff time.Time +} + +func (s *reconcileRun) note(f *Finding) { + if f == nil { + return + } + s.stats.Findings++ + if f.Applied { + s.stats.Repaired++ + } + if s.emit != nil { + s.emit(*f) + } +} + +func (s *reconcileRun) old(modified time.Time) bool { + // Unknown timestamps must never authorize collection. + return !modified.IsZero() && modified.Before(s.cutoff) +} + +func (s *reconcileRun) objects(prefix string) error { + after := "" + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + page, err := s.r.blobs.List(opCtx, prefix, after, reconcilePageSize) + cancel() + if err != nil { + return fmt.Errorf("reconcile list %s: %w", prefix, err) + } + if len(page) == 0 { + return nil + } + var known map[string]bool + if prefix == "blobs/" { + known, err = s.knownFiles(page) + } else { + known, err = s.knownJobObjects(s.ctx, s.r.db, prefix, page) + } + if err != nil { + return err + } + for _, obj := range page { + if obj.Key <= after { + return errors.New("reconcile: object page did not advance") + } + after = obj.Key + s.stats.ScannedObjects++ + if !s.old(obj.Modified) || known[obj.Key] { + continue + } + var finding *Finding + if prefix == "blobs/" { + finding, err = s.orphan(obj) + } else { + finding, err = s.jobObject(prefix, obj) + } + if err != nil { + return fmt.Errorf("reconcile %s: %w", obj.Key, err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) stat(key string) (Object, error) { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + return s.r.blobs.Stat(opCtx, key) +} + +func (s *reconcileRun) delete(key string) error { + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + err := s.r.blobs.Delete(opCtx, key) + if errors.Is(err, ErrObjectMissing) { + return nil + } + return err +} + +func parseJobObject(prefix, key string) (uuid.UUID, int32, bool) { + parts := strings.Split(strings.TrimPrefix(key, prefix), "/") + id, err := uuid.Parse(parts[0]) + if prefix == "staging/" { + return id, 0, err == nil + } + if len(parts) != 4 || err != nil { + return uuid.Nil, 0, false + } + version, err := strconv.ParseInt(parts[1], 10, 32) + return id, int32(version), err == nil && version > 0 +} + +type jobObjectReader interface { + Query(context.Context, string, ...any) (pgx.Rows, error) +} + +// The same lookup filters an S3 page and rechecks a single candidate inside its deletion transaction. +func (s *reconcileRun) knownJobObjects(ctx context.Context, db jobObjectReader, prefix string, page []Object) (map[string]bool, error) { + var ids []uuid.UUID + var keys []string + var versions []int32 + for _, obj := range page { + if !s.old(obj.Modified) { + continue + } + id, version, ok := parseJobObject(prefix, obj.Key) + if !ok { + continue + } + ids = append(ids, id) + keys = append(keys, obj.Key) + if prefix == "variants/" { + versions = append(versions, version) + } + } + if len(keys) == 0 { + return nil, nil + } + var query string + var args []any + if prefix == "variants/" { + query = `SELECT o.key FROM unnest($1::uuid[], $2::text[], $3::integer[]) AS o(id,key,version) + WHERE EXISTS (SELECT 1 FROM attachment_variants v + WHERE v.attachment_id=o.id AND v.pipeline_version=o.version AND v.object_key=o.key) + OR EXISTS (SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id=o.id AND j.pipeline_version=o.version AND j.state IN ('queued','processing'))` + args = []any{ids, keys, versions} + } else { + query = `SELECT o.key FROM unnest($1::uuid[], $2::text[]) AS o(id,key) + WHERE EXISTS (SELECT 1 FROM attachment_jobs j WHERE j.attachment_id=o.id AND j.source_key=o.key AND + (j.state IN ('queued','processing') OR (j.state='failed' AND j.updated_at >= now()-$3::bigint*interval '1 second')))` + args = []any{ids, keys, int64(s.r.retention / time.Second)} + } + s.stats.SQLQueries++ + rows, err := db.Query(ctx, query, args...) + if err != nil { + return nil, err + } + known := make(map[string]bool) + var key string + _, err = pgx.ForEachRow(rows, []any{&key}, func() error { + known[key] = true + return nil + }) + return known, err +} + +func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) { + id, _, valid := parseJobObject(prefix, obj.Key) + kind := "stale_staging" + if prefix == "variants/" { + if !valid { + return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil + } + kind = "orphan_variant" + } + finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"} + if !s.apply { + return finding, nil + } + keyHash := sha256.Sum256([]byte(obj.Key)) + return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + // Intake locks the staging key before inserting an attachment. Replay, + // reprocess and completion lock the existing attachment before changing job state. + if valid { + s.stats.SQLQueries++ + if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil { + return nil, err + } + } + known, err := s.knownJobObjects(opCtx, tx, prefix, []Object{obj}) + if err != nil { + return nil, err + } + if known[obj.Key] { + return nil, nil + } + op := *s + op.ctx = opCtx + current, err := op.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err := op.delete(obj.Key); err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + }) +} + +func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { + hashes := make([][]byte, 0, len(page)) + for _, obj := range page { + if !s.old(obj.Modified) { + continue + } + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err == nil && len(hash) == sha256.Size && BlobKey(hash) == obj.Key { + hashes = append(hashes, hash) + } + } + known := make(map[string]bool, len(hashes)) + if len(hashes) == 0 { + return known, nil + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, ` + SELECT hash FROM files WHERE hash = ANY($1::bytea[])`, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var hash []byte + if err := rows.Scan(&hash); err != nil { + return nil, err + } + known[BlobKey(hash)] = true + } + return known, rows.Err() +} + +func (s *reconcileRun) orphan(obj Object) (*Finding, error) { + hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) + if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key { + return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil + } + f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"} + if !s.apply { + return f, nil + } + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var exists bool + s.stats.SQLQueries++ + if err := tx.QueryRow(opCtx, ` + SELECT EXISTS (SELECT 1 FROM files WHERE hash = $1)`, hash).Scan(&exists); err != nil { + return nil, err + } + if exists { + return nil, nil + } + current, err := op.stat(obj.Key) + if errors.Is(err, ErrObjectMissing) { + return nil, nil + } + if err != nil { + return nil, err + } + if !s.old(current.Modified) { + return nil, nil + } + if err = op.delete(obj.Key); err != nil { + return nil, err + } + f.Applied = true + return f, nil + }) +} + +// locked also covers hashes without a files row: a publisher and an orphan +// collector therefore cannot pass each other's existence check concurrently. +func (s *reconcileRun) locked(hash []byte, fn func(context.Context, pgx.Tx) (*Finding, error)) (*Finding, error) { + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + s.stats.SQLQueries++ + tx, err := s.r.db.Begin(opCtx) + if err != nil { + return nil, err + } + committed := false + defer func() { + if !committed { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s.stats.SQLQueries++ + _ = tx.Rollback(cleanup) + } + }() + // Pool defaults cover ordinary requests; object I/O can outlive those + // idle limits. The operation deadline bounds the entire locked candidate. + s.stats.SQLQueries++ + if _, err = tx.Exec(opCtx, `SET LOCAL idle_in_transaction_session_timeout='35s'`); err != nil { + return nil, err + } + s.stats.SQLQueries++ + if err = LockHash(opCtx, tx, hash); err != nil { + return nil, err + } + f, err := fn(opCtx, tx) + if err != nil { + return nil, err + } + s.stats.SQLQueries++ + err = tx.Commit(opCtx) + committed = true // pgx closes the transaction even when COMMIT fails. + if err != nil { + return nil, err + } + return f, nil +} + +type reconcileFile struct { + hash []byte + size int64 + status FileStatus + created time.Time + referenced bool +} + +func (s *reconcileRun) files() error { + after := []byte{} + for { + if err := s.ctx.Err(); err != nil { + return err + } + s.stats.SQLQueries++ + rows, err := s.r.db.Query(s.ctx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash > $1 + ORDER BY f.hash + LIMIT $2`, after, reconcilePageSize) + if err != nil { + return err + } + page := make([]reconcileFile, 0, reconcilePageSize) + for rows.Next() { + var f reconcileFile + if err = rows.Scan(&f.hash, &f.size, &f.status, &f.created, &f.referenced); err != nil { + rows.Close() + return err + } + page = append(page, f) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + // Close the page's SQL rows before acquiring a transaction connection. + for _, f := range page { + after = f.hash + s.stats.ScannedFiles++ + if !s.candidate(f) { + continue + } + finding, err := s.inspect(f) + if err == nil && s.apply && finding != nil && finding.Action != "report" { + finding, err = s.repairFile(f.hash) + } + if err != nil { + return fmt.Errorf("reconcile file %s: %w", BlobKey(f.hash), err) + } + s.note(finding) + } + } +} + +func (s *reconcileRun) repairFile(hash []byte) (*Finding, error) { + return s.locked(hash, func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { + op := *s + op.ctx = opCtx + var current reconcileFile + s.stats.SQLQueries++ + err := tx.QueryRow(opCtx, ` + SELECT f.hash, f.size_bytes, f.status, f.created_at, + EXISTS (SELECT 1 FROM attachments a WHERE a.file_hash = f.hash) + FROM files f + WHERE f.hash = $1 + FOR UPDATE OF f`, hash). + Scan(¤t.hash, ¤t.size, ¤t.status, ¤t.created, ¤t.referenced) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if !op.candidate(current) { + return nil, nil + } + finding, err := op.inspect(current) + if err != nil || finding == nil { + return finding, err + } + switch finding.Action { + case "delete_file": + // Delete bytes first. A failed SQL commit leaves an unreferenced + // missing row that the next pass can safely finish collecting. + if err := op.delete(finding.Key); err != nil { + return nil, err + } + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `DELETE FROM files WHERE hash = $1`, hash) + case "mark_failed": + s.stats.SQLQueries++ + _, err = tx.Exec(opCtx, `UPDATE files SET status = $2 WHERE hash = $1`, hash, FileFailed) + default: + return finding, nil + } + if err != nil { + return nil, err + } + finding.Applied = true + return finding, nil + }) +} + +func (s *reconcileRun) candidate(f reconcileFile) bool { + if !s.old(f.created) { + return false + } + if !f.referenced { + return f.status == FilePending || f.status == FileReady + } + return f.status == FileReady || f.status == FileFailed +} + +func (s *reconcileRun) inspect(f reconcileFile) (*Finding, error) { + key := BlobKey(f.hash) + if !f.referenced { + // A publication that rolled back can leave fresh bytes behind an old + // pending row. Both the metadata and the current object must be old. + obj, err := s.stat(key) + if err != nil && !errors.Is(err, ErrObjectMissing) { + return nil, err + } + if err == nil && !s.old(obj.Modified) { + return nil, nil + } + return &Finding{Kind: "unreferenced_" + string(f.status), Key: key, Action: "delete_file"}, nil + } + obj, err := s.stat(key) + if errors.Is(err, ErrObjectMissing) { + if f.status == FileFailed { + return nil, nil + } + return &Finding{Kind: "missing_blob", Key: key, Action: "mark_failed"}, nil + } + if err != nil { + return nil, err + } + if obj.Size != f.size { + return &Finding{Kind: "size_mismatch", Key: key, Action: "report"}, nil + } + if s.deep { + match, err := s.verify(f) + if err != nil { + return nil, err + } + if !match { + return &Finding{Kind: "hash_mismatch", Key: key, Action: "report"}, nil + } + } + // A failed file stays failed even when the key reappears. Recovery requires + // an explicit publisher; a shallow HEAD cannot prove content integrity. + if f.status == FileFailed { + return &Finding{Kind: "failed_present", Key: key, Action: "report"}, nil + } + return nil, nil +} + +func (s *reconcileRun) verify(f reconcileFile) (bool, error) { + if f.size < 0 || f.size == math.MaxInt64 { + return false, errors.New("reconcile: invalid file size") + } + s.stats.S3Calls++ + opCtx, cancel := context.WithTimeout(s.ctx, reconcileOperationTimeout) + defer cancel() + body, err := s.r.blobs.Open(opCtx, BlobKey(f.hash)) + if err != nil { + return false, err + } + hash := sha256.New() + n, readErr := io.Copy(hash, io.LimitReader(body, f.size+1)) + closeErr := body.Close() + if readErr != nil { + return false, readErr + } + if closeErr != nil { + return false, closeErr + } + return n == f.size && bytes.Equal(hash.Sum(nil), f.hash), nil +} diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go new file mode 100644 index 00000000..674aed57 --- /dev/null +++ b/internal/attachment/reconcile_test.go @@ -0,0 +1,844 @@ +package attachment_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func TestReconcileBlobKey(t *testing.T) { + hash, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + if got := attachment.BlobKey(hash); got != "blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Fatalf("blob key: %q", got) + } +} + +// Only object storage is doubled: SQL, transactions, FKs and advisory locks are real. +type reconcileBlobs struct { + mu sync.Mutex + objects map[string]attachment.Object + bodies map[string][]byte + statErr, deleteErr, openErr error + listed func(string) + stating func(string) + opening func(string) io.ReadCloser + writes, readBytes int + limits []int +} + +func newReconcileBlobs() *reconcileBlobs { + return &reconcileBlobs{objects: make(map[string]attachment.Object), bodies: make(map[string][]byte)} +} +func (b *reconcileBlobs) add(key string, body []byte, modified time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + b.objects[key] = attachment.Object{Key: key, Size: int64(len(body)), Modified: modified} + b.bodies[key] = bytes.Clone(body) +} +func (b *reconcileBlobs) has(key string) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.objects[key] + return ok +} +func (b *reconcileBlobs) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error { + body, err := io.ReadAll(io.LimitReader(r, size+1)) + if err != nil { + return err + } + if int64(len(body)) != size { + return io.ErrUnexpectedEOF + } + b.add(key, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Copy(_ context.Context, source, dest string) error { + b.mu.Lock() + body, ok := b.bodies[source] + b.mu.Unlock() + if !ok { + return attachment.ErrObjectMissing + } + b.add(dest, body, time.Now()) + b.mu.Lock() + b.writes++ + b.mu.Unlock() + return nil +} +func (b *reconcileBlobs) Open(_ context.Context, key string) (io.ReadCloser, error) { + if b.openErr != nil { + return nil, b.openErr + } + if b.opening != nil { + return b.opening(key), nil + } + b.mu.Lock() + defer b.mu.Unlock() + body, ok := b.bodies[key] + if !ok { + return nil, attachment.ErrObjectMissing + } + return io.NopCloser(bytes.NewReader(body)), nil +} +func (b *reconcileBlobs) Stat(_ context.Context, key string) (attachment.Object, error) { + if b.stating != nil { + b.stating(key) + } + b.mu.Lock() + defer b.mu.Unlock() + if b.statErr != nil { + return attachment.Object{}, b.statErr + } + obj, ok := b.objects[key] + if !ok { + return attachment.Object{}, fmt.Errorf("head: %w", attachment.ErrObjectMissing) + } + return obj, nil +} +func (b *reconcileBlobs) Delete(_ context.Context, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.deleteErr != nil { + return b.deleteErr + } + delete(b.objects, key) + delete(b.bodies, key) + b.writes++ + return nil +} +func (b *reconcileBlobs) List(_ context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + b.mu.Lock() + b.limits = append(b.limits, limit) + if limit < 1 || limit > 500 { + b.mu.Unlock() + return nil, errors.New("unbounded page") + } + var page []attachment.Object + for key, obj := range b.objects { + if strings.HasPrefix(key, prefix) && key > after { + page = append(page, obj) + } + } + sort.Slice(page, func(i, j int) bool { return page[i].Key < page[j].Key }) + if len(page) > limit { + page = page[:limit] + } + b.mu.Unlock() + if b.listed != nil { + b.listed(prefix) + } + return page, nil +} + +type reconcileFixture struct { + t *testing.T + db *pgxpool.Pool + b *reconcileBlobs + user, chat, message uuid.UUID +} + +func reconcileSetup(t *testing.T, db *testenv.DB) *reconcileFixture { + t.Helper() + db.Reset(t) + f := &reconcileFixture{t: t, db: db.Pool, b: newReconcileBlobs(), user: uuid.New(), chat: uuid.New(), message: uuid.New()} + f.exec(`INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,'reconcile','Reconcile','unused',now())`, f.user) + f.exec(`INSERT INTO chats(id,title,created_at) VALUES($1,'reconcile',now())`, f.chat) + f.exec(`INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, f.message, f.user) + return f +} +func (f *reconcileFixture) exec(sql string, args ...any) { + f.t.Helper() + if _, err := f.db.Exec(context.Background(), sql, args...); err != nil { + f.t.Fatal(err) + } +} +func (f *reconcileFixture) file(body, status string, age time.Duration, referenced, present bool) []byte { + f.t.Helper() + hash := sha256.Sum256([]byte(body)) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, time.Now().Add(-age)) + if referenced { + id := uuid.New() + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'image.png',$4,now())`, id, f.chat, f.user, hash[:]) + f.exec(`INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) + VALUES($1,$2,$3,(SELECT count(*) FROM message_attachments WHERE chat_id=$1 AND message_id=$2))`, f.chat, f.message, id) + } + if present { + f.b.add(attachment.BlobKey(hash[:]), []byte(body), time.Now().Add(-age)) + } + return hash[:] +} +func (f *reconcileFixture) status(hash []byte) string { + f.t.Helper() + var s string + err := f.db.QueryRow(context.Background(), `SELECT status FROM files WHERE hash=$1`, hash).Scan(&s) + if errors.Is(err, pgx.ErrNoRows) { + return "" + } + if err != nil { + f.t.Fatal(err) + } + return s +} +func (f *reconcileFixture) run(apply, deep bool) (attachment.Stats, []attachment.Finding, error) { + f.t.Helper() + r, err := attachment.NewReconciler(f.db, f.b, 0) + if err != nil { + f.t.Fatal(err) + } + var found []attachment.Finding + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(context.Background(), options, func(v attachment.Finding) { found = append(found, v) }) + return stats, found, err +} + +func TestReconciler(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("healthy apply does not block publication", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("healthy", "ready", 2*time.Hour, true, true) + tx, err := db.Pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err := attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, f.b, time.Hour) + if err != nil { + t.Fatal(err) + } + scanCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + stats, err := r.Run(scanCtx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) + if err != nil || stats.Findings != 0 || stats.Repaired != 0 { + t.Fatalf("healthy inspection waited for publication: %+v %v", stats, err) + } + }) + t.Run("constructor validates and defaults grace", func(t *testing.T) { + f := reconcileSetup(t, db) + for _, grace := range []time.Duration{-time.Second, time.Minute} { + if _, err := attachment.NewReconciler(f.db, f.b, grace); err == nil { + t.Fatalf("accepted grace %v", grace) + } + } + if _, err := attachment.NewReconciler(nil, f.b, 0); err == nil { + t.Fatal("accepted nil DB") + } + if _, err := attachment.NewReconciler(f.db, nil, 0); err == nil { + t.Fatal("accepted nil blobs") + } + if _, err := attachment.NewReconciler(f.db, f.b, 2*time.Minute); err != nil { + t.Fatal(err) + } + fresh := f.file("half hour", "ready", 30*time.Minute, false, true) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(fresh) != "ready" { + t.Fatalf("default grace: %+v %v", stats, err) + } + }) + t.Run("dry run then apply then repeat preserves fresh referenced and message links", func(t *testing.T) { + f := reconcileSetup(t, db) + old := 2 * time.Hour + pending := f.file("pending", "pending", old, false, true) + unused := f.file("unused", "ready", old, false, true) + used := f.file("used", "ready", old, true, true) + fresh := f.file("fresh", "pending", time.Minute, false, true) + missing := f.file("missing", "ready", old, true, false) + f.b.add("staging/old", []byte("old"), time.Now().Add(-old)) + f.b.add("staging/fresh", []byte("new"), time.Now()) + orphan := sha256.Sum256([]byte("orphan")) + orphanKey := attachment.BlobKey(orphan[:]) + f.b.add(orphanKey, []byte("orphan"), time.Now().Add(-old)) + stats, findings, err := f.run(false, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 5 || len(findings) != 5 { + t.Fatalf("dry: %+v %v %+v", stats, err, findings) + } + if f.b.writes != 0 || f.status(pending) != "pending" || f.status(missing) != "ready" { + t.Fatal("dry run mutated storage") + } + for _, finding := range findings { + if finding.Applied { + t.Fatal("dry finding marked applied") + } + } + stats, findings, err = f.run(true, false) + if err != nil || stats.Repaired != 5 { + t.Fatalf("apply: %+v %v %+v", stats, err, findings) + } + if f.status(pending) != "" || f.status(unused) != "" || f.status(missing) != "failed" { + t.Fatal("metadata repair incorrect") + } + if f.b.has(orphanKey) || f.b.has("staging/old") || !f.b.has("staging/fresh") || !f.b.has(attachment.BlobKey(used)) || !f.b.has(attachment.BlobKey(fresh)) { + t.Fatal("incorrect object collection") + } + var links int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM message_attachments`).Scan(&links); err != nil || links != 2 { + t.Fatalf("message links lost: %d %v", links, err) + } + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 { + t.Fatalf("repeat: %+v %v", stats, err) + } + }) + t.Run("outage is not missing and failed delete keeps row", func(t *testing.T) { + f := reconcileSetup(t, db) + used := f.file("used", "ready", 2*time.Hour, true, true) + outage := errors.New("injected S3 outage") + f.b.statErr = outage + stats, _, err := f.run(true, false) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(used) != "ready" { + t.Fatalf("outage: %+v %v", stats, err) + } + f.b.statErr = nil + unused := f.file("unused", "pending", 2*time.Hour, false, true) + f.b.deleteErr = outage + _, _, err = f.run(true, false) + if !errors.Is(err, outage) || f.status(unused) != "pending" { + t.Fatalf("delete failure: %v", err) + } + }) + t.Run("deep reports corruption and never revives failed without verification", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("evil"), time.Now().Add(-2*time.Hour)) + stats, found, err := f.run(true, true) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("deep: %+v %v %+v", stats, err, found) + } + f.exec(`UPDATE files SET status='failed' WHERE hash=$1`, hash) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("shallow revived failed file") + } + stats, _, err = f.run(true, true) + if err != nil || stats.Repaired != 0 || f.status(hash) != "failed" { + t.Fatal("corrupt file revived") + } + }) + t.Run("wrong size reports only and fresh missing stays ready", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("sized", "ready", 2*time.Hour, true, true) + f.b.add(attachment.BlobKey(hash), []byte("longer body"), time.Now().Add(-2*time.Hour)) + fresh := f.file("new missing", "ready", time.Minute, true, false) + stats, found, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || stats.Findings != 1 || found[0].Kind != "size_mismatch" || f.status(fresh) != "ready" { + t.Fatalf("size/fresh: %+v %v %+v", stats, err, found) + } + }) + t.Run("orphan rechecks current object age", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("orphan")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("orphan"), time.Now().Add(-2*time.Hour)) + f.b.listed = func(prefix string) { + if prefix == "blobs/" { + f.b.add(key, []byte("republished"), time.Now()) + } + } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || !f.b.has(key) { + t.Fatalf("fresh replacement deleted: %+v %v", stats, err) + } + }) + t.Run("row rechecks references after taking lock", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("race refs", "ready", 2*time.Hour, false, true) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + _, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'race.png',$4,now())`, uuid.New(), f.chat, f.user, hash) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if f.status(hash) != "ready" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatal("new reference collected") + } + }) + t.Run("concurrent upload lock prevents orphan collection", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := sha256.Sum256([]byte("publishing")) + key := attachment.BlobKey(hash[:]) + f.b.add(key, []byte("publishing"), time.Now().Add(-2*time.Hour)) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash[:]); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + r, _ := attachment.NewReconciler(f.db, f.b, 0) + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + if _, err = tx.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES($1,10,'image/png','ready')`, hash[:]); err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; err != nil { + t.Fatal(err) + } + if !f.b.has(key) || f.status(hash[:]) != "ready" { + t.Fatal("published blob collected") + } + }) + t.Run("signed big endian lock key matches publisher", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := make([]byte, 32) + for i := 0; i < 8; i++ { + hash[i] = 255 + } + hash[7] = 254 + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + if err = attachment.LockHash(ctx, tx, hash); err != nil { + t.Fatal(err) + } + other, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer other.Rollback(ctx) + var acquired bool + if err = other.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(-2::bigint)`).Scan(&acquired); err != nil || acquired { + t.Fatalf("lock mismatch: %v %v", acquired, err) + } + }) + t.Run("keyset pages tolerate deletes and report costs", func(t *testing.T) { + f := reconcileSetup(t, db) + for i := 0; i < 1003; i++ { + f.b.add(fmt.Sprintf("staging/%04d", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','pending',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + stats, _, err := f.run(true, false) + if err != nil || stats.ScannedFiles != 1003 || stats.ScannedObjects != 1003 || stats.Repaired != 2006 { + t.Fatalf("pages: %+v %v", stats, err) + } + var n int + if err := f.db.QueryRow(ctx, `SELECT count(*) FROM files`).Scan(&n); err != nil || n != 0 { + t.Fatalf("skipped rows: %d %v", n, err) + } + if len(f.b.limits) < 4 || stats.SQLQueries == 0 || stats.S3Calls == 0 { + t.Fatalf("missing paging/costs: %+v", stats) + } + raw, err := json.Marshal(stats) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"elapsed_ms", "sql_queries", "s3_calls", "scanned_files", "scanned_objects", "findings", "repaired"} { + if !bytes.Contains(raw, []byte(`"`+key+`":`)) { + t.Fatalf("missing JSON key %s: %s", key, raw) + } + } + }) + t.Run("old blob existence checks are batched by page", func(t *testing.T) { + f := reconcileSetup(t, db) + f.exec(`INSERT INTO files(hash,size_bytes,mime_type,status,created_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),1,'image/png','ready',now()-interval '2 hour' FROM generate_series(1,1003) AS i`) + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),$1,$2,'batch.png',hash,now() FROM files`, f.chat, f.user) + for i := 1; i <= 1003; i++ { + f.b.add(fmt.Sprintf("blobs/%064x", i), []byte("x"), time.Now().Add(-2*time.Hour)) + } + stats, _, err := f.run(false, false) + if err != nil || stats.SQLQueries > 8 || stats.SQLQueries == 0 || stats.ScannedFiles != 1003 || stats.Findings != 0 { + t.Fatalf("per-object SQL lookup: %+v %v", stats, err) + } + }) + t.Run("published variants and active staging sources are checked by page", func(t *testing.T) { + f := reconcileSetup(t, db) + const count = 100 + old := time.Now().Add(-2 * time.Hour) + digest := strings.Repeat("0", 64) + hash := f.file("shared original", "ready", 2*time.Hour, false, true) + ids := make([]uuid.UUID, count) + for i := range ids { + id := uuid.New() + ids[i] = id + f.b.add("staging/"+id.String(), []byte("x"), old) + for _, name := range []string{"thumbnail", "crop", "optimized"} { + key := fmt.Sprintf("variants/%s/1/%s/%s", id, name, digest) + f.b.add(key, []byte("x"), old) + } + } + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at,target_version,published_version) + SELECT id,$2,$3,'batch.png',$4,now(),2,1 FROM unnest($1::uuid[]) id`, ids, f.chat, f.user, hash) + f.exec(`INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key,state) + SELECT id,1,'staging/'||id,'ready' FROM attachments UNION ALL + SELECT id,2,'staging/'||id,'queued' FROM attachments`) + f.exec(`INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256) + SELECT id,1,name,'variants/'||id||'/1/'||name||'/'||$1, + CASE WHEN name='optimized' THEN 'image/webp' ELSE 'image/jpeg' END,1,1,1,decode($1,'hex') + FROM attachments CROSS JOIN unnest(ARRAY['thumbnail','crop','optimized']) name`, digest) + stats, _, err := f.run(true, false) + t.Logf("%d published variants and %d active staging sources: %+v", 3*count, count, stats) + if err != nil || stats.Findings != 0 || stats.ScannedObjects != 4*count+1 || stats.SQLQueries > 6 { + t.Fatalf("expected page-batched lookups to preserve all referenced objects: %+v, %v", stats, err) + } + }) + t.Run("new job protects a variant after the page lookup", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("pending source", "pending", 2*time.Hour, true, false) + var id uuid.UUID + if err := f.db.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil { + t.Fatal(err) + } + key := fmt.Sprintf("variants/%s/1/thumbnail/%064x", id, 0) + f.b.add(key, []byte("x"), time.Now().Add(-2*time.Hour)) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + keyHash := sha256.Sum256([]byte(key)) + if err := attachment.LockHash(ctx, tx, keyHash[:]); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, f.b, 0) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + if _, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id, source_key) VALUES ($1,$2)`, id, "staging/"+id.String()); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil || !f.b.has(key) { + t.Fatalf("variant required by new job was lost: %v", err) + } + }) +} + +func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) { + t.Helper() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + for { + var waiting bool + if err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted)`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + return + } + select { + case <-deadline.C: + t.Fatal("reconciler did not wait on publisher advisory lock") + case <-tick.C: + } + } +} + +type reconcileEndless struct { + read int64 + closed bool +} + +func (r *reconcileEndless) Read(p []byte) (int, error) { + for i := range p { + p[i] = 'x' + } + r.read += int64(len(p)) + return len(p), nil +} +func (r *reconcileEndless) Close() error { r.closed = true; return nil } + +func TestReconcilerFailureBoundaries(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + t.Run("slow object call survives pool idle transaction timeout", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("slow object", "pending", 2*time.Hour, false, true) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "100ms" + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + f.db = pool + f.b.stating = func(string) { time.Sleep(250 * time.Millisecond) } + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("slow S3 transaction rolled back: %+v %v", stats, err) + } + }) + t.Run("fresh object with old unused metadata is preserved", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("republished", "pending", 2*time.Hour, false, true) + f.b.add(attachment.BlobKey(hash), []byte("republished"), time.Now()) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 0 || f.status(hash) != "pending" || !f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("fresh blob lost: %+v %v", stats, err) + } + }) + t.Run("deep read stops at size plus one and closes body", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + body := &reconcileEndless{} + f.b.opening = func(string) io.ReadCloser { return body } + stats, found, err := f.run(true, true) + if err != nil || body.read != 5 || !body.closed || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "hash_mismatch" || f.status(hash) != "ready" { + t.Fatalf("unbounded deep read: read=%d closed=%v %+v %v", body.read, body.closed, stats, err) + } + }) + t.Run("deep GET outage preserves referenced state", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, true) + outage := errors.New("injected GET outage") + f.b.openErr = outage + stats, _, err := f.run(true, true) + if !errors.Is(err, outage) || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("GET outage: %+v %v", stats, err) + } + }) + t.Run("valid failed object remains stable even deep", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("good", "failed", 2*time.Hour, true, true) + for _, deep := range []bool{false, true} { + stats, found, err := f.run(true, deep) + if err != nil || stats.Repaired != 0 || len(found) != 1 || found[0].Kind != "failed_present" || f.status(hash) != "failed" { + t.Fatalf("failed state unstable: %+v %v %+v", stats, err, found) + } + } + }) + t.Run("dry run works with a read only PostgreSQL session", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("missing", "ready", 2*time.Hour, true, false) + cfg := f.db.Config() + cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + readOnly, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer readOnly.Close() + r, err := attachment.NewReconciler(readOnly, f.b, 0) + if err != nil { + t.Fatal(err) + } + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, nil) + if err != nil || stats.Findings != 1 || stats.Repaired != 0 || f.status(hash) != "ready" { + t.Fatalf("read only dry run: %+v %v", stats, err) + } + }) + t.Run("failed SQL commit after blob deletion is retryable", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("unused", "pending", 2*time.Hour, false, true) + f.exec(`CREATE FUNCTION reconcile_reject_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected commit failure'; END $$`) + f.exec(`CREATE CONSTRAINT TRIGGER reconcile_reject_delete AFTER DELETE ON files DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reconcile_reject_delete()`) + t.Cleanup(func() { + f.exec(`DROP TRIGGER IF EXISTS reconcile_reject_delete ON files`) + f.exec(`DROP FUNCTION IF EXISTS reconcile_reject_delete()`) + }) + stats, _, err := f.run(true, false) + if err == nil || stats.Repaired != 0 || f.status(hash) != "pending" || f.b.has(attachment.BlobKey(hash)) { + t.Fatalf("failed commit: %+v %v", stats, err) + } + f.exec(`DROP TRIGGER reconcile_reject_delete ON files`) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 1 || f.status(hash) != "" { + t.Fatalf("retry: %+v %v", stats, err) + } + }) +} + +func TestReconcilerS3(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s3.Close() + f := reconcileSetup(t, db) + hash := f.file("good", "ready", 2*time.Hour, true, false) + missing := f.file("missing", "ready", 2*time.Hour, true, false) + key := attachment.BlobKey(hash) + if err = s3.Store.Put(ctx, key, strings.NewReader("evil"), 4, "image/png"); err != nil { + t.Fatal(err) + } + if err = s3.Store.Put(ctx, "staging/fresh", strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, s3.Store, 0) + if err != nil { + t.Fatal(err) + } + // Exercise the command with file-based configuration and the real listing + // identity. Its default must remain read-only, including missing metadata. + secretDir := t.TempDir() + cliEnv := os.Environ() + for name, value := range map[string]string{ + "DATABASE_URL_FILE": db.DSN, + "S3_ACCESS_KEY_ID_FILE": s3.AccessKey, + "S3_SECRET_ACCESS_KEY_FILE": s3.SecretKey, + } { + path := filepath.Join(secretDir, name) + if err := os.WriteFile(path, []byte(value), 0600); err != nil { + t.Fatal(err) + } + cliEnv = append(cliEnv, name+"="+path) + } + cliEnv = append(cliEnv, "S3_ENDPOINT="+s3.Endpoint, "S3_REGION="+s3.Region, "S3_BUCKET="+s3.Bucket) + cmd := exec.CommandContext(ctx, "go", "run", "../../cmd/reconcile", "--deep") + cmd.Env = cliEnv + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("CLI dry run: %v: %s", err, stderr.String()) + } + decoder := json.NewDecoder(&stdout) + var cliStats attachment.Stats + for i := 0; i < 2; i++ { + var finding attachment.Finding + if err := decoder.Decode(&finding); err != nil || finding.Applied || finding.Kind == "" || finding.Key == "" || finding.Action == "" { + t.Fatalf("CLI finding: %+v %v", finding, err) + } + } + if err := decoder.Decode(&cliStats); err != nil || cliStats.Findings != 2 || cliStats.Repaired != 0 || f.status(missing) != "ready" { + t.Fatalf("CLI mutated by default: %+v %v", cliStats, err) + } + var found []attachment.Finding + stats, err := r.Run(ctx, attachment.ReconcileOptions{Deep: true}, func(v attachment.Finding) { found = append(found, v) }) + if err != nil || stats.Repaired != 0 || stats.Findings != 2 || f.status(missing) != "ready" { + t.Fatalf("S3 dry: %+v %v %+v", stats, err, found) + } + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) + if err != nil || stats.Repaired != 1 || f.status(missing) != "failed" || f.status(hash) != "ready" { + t.Fatalf("S3 apply: %+v %v", stats, err) + } + if _, err = s3.Store.Stat(ctx, key); err != nil { + t.Fatal("referenced corrupt object deleted", err) + } + if _, err = s3.Store.Stat(ctx, "staging/fresh"); err != nil { + t.Fatal("fresh staging deleted", err) + } + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true, Deep: true}, nil) + if err != nil || stats.Repaired != 0 { + t.Fatalf("S3 repeat: %+v %v", stats, err) + } +} + +func TestReconcileLockRejectsMalformedHash(t *testing.T) { + if err := attachment.LockHash(context.Background(), nil, []byte("bad")); err == nil { + t.Fatal("malformed hash accepted") + } +} +func TestGCRespectsJobSourcesAndVersionedOutputs(t *testing.T) { + ctx := context.Background() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + f := reconcileSetup(t, db) + hash := f.file("job input", "pending", 48*time.Hour, true, false) + var id uuid.UUID + if err := db.Pool.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil { + t.Fatal(err) + } + stage := "staging/" + id.String() + output := "variants/" + id.String() + "/1/thumbnail/" + strings.Repeat("0", 64) + old := time.Now().Add(-48 * time.Hour) + f.b.add(stage, []byte("job input"), old) + f.b.add(output, []byte("partial"), old) + f.exec(`INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, stage) + for _, apply := range []bool{false, true} { + stats, _, err := f.run(apply, false) + if err != nil || stats.Findings != 0 || !f.b.has(stage) || !f.b.has(output) { + t.Fatalf("active job lost: %+v %v", stats, err) + } + } + f.exec(`UPDATE attachment_jobs SET state='failed', error_code='retry_exhausted',updated_at=now()`) + stats, _, err := f.run(true, false) + if err != nil || stats.Repaired != 1 || !f.b.has(stage) || f.b.has(output) { + t.Fatalf("failed retention: %+v %v", stats, err) + } + f.exec(`UPDATE attachment_jobs SET updated_at=now()-interval '3 hours'`) + stats, _, err = f.run(true, false) + if err != nil || stats.Repaired != 0 || !f.b.has(stage) { + t.Fatalf("default failed retention: %+v %v", stats, err) + } + r, err := attachment.NewReconcilerWithRetention(db.Pool, f.b, time.Hour, 2*time.Hour) + if err != nil { + t.Fatal(err) + } + stats, err = r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + if err != nil || stats.Repaired != 1 || f.b.has(stage) { + t.Fatalf("expired source: %+v %v", stats, err) + } +} diff --git a/internal/attachment/repository.go b/internal/attachment/repository.go new file mode 100644 index 00000000..bae447b1 --- /dev/null +++ b/internal/attachment/repository.go @@ -0,0 +1,17 @@ +package attachment + +import ( + "context" + + "github.com/google/uuid" +) + +type AttachmentRepository interface { + // Enqueue checks the staging object under the collector's lock before committing its job. + Enqueue(ctx context.Context, attachment Attachment, stage string, checkStage func() error) error + GetForMember(ctx context.Context, attachmentID, actorID uuid.UUID) (Attachment, error) +} + +type ChatMembership interface { + IsMember(ctx context.Context, chatID, actorID uuid.UUID) (bool, error) +} diff --git a/internal/attachment/schema_test.go b/internal/attachment/schema_test.go new file mode 100644 index 00000000..c5db7ee6 --- /dev/null +++ b/internal/attachment/schema_test.go @@ -0,0 +1,27 @@ +package attachment_test + +import ( + "context" + "messenger/internal/testenv" + "testing" +) + +func TestFilesSchemaRejectsMalformedHashes(t *testing.T) { + db, err := testenv.StartPostgres(context.Background()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err != nil { + t.Fatalf("valid content-addressed file: %v", err) + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ('bad',12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted a malformed SHA256") + } + _, err = db.Pool.Exec(context.Background(), `INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),12,'image/png','ready')`) + if err == nil { + t.Fatal("accepted duplicate content hash") + } +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 00000000..e3417f47 --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,245 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "log/slog" + "net/http" + "path/filepath" + "time" + + "github.com/google/uuid" + "messenger/internal/imaging" + "messenger/internal/platform/apperr" + "messenger/internal/platform/faultbarrier" + "messenger/internal/platform/ident" +) + +type Service struct { + attachments AttachmentRepository + chats ChatMembership + blobs Blobs + config Config + decode chan struct{} +} + +type Config struct { + MaxBytes int64 + MaxPixels int64 + BarrierDir string +} + +// A nil blobs dependency allows the text API to run without configured S3. +// Binary operations report unavailability in that mode. +func NewService(attachments AttachmentRepository, chats ChatMembership, blobs Blobs, cfg Config) *Service { + if cfg.MaxBytes == 0 { + cfg.MaxBytes = imaging.MaxUploadBytes + } + if cfg.MaxPixels == 0 { + cfg.MaxPixels = imaging.MaxImagePixels + } + return &Service{ + attachments: attachments, + chats: chats, + blobs: blobs, + config: cfg, + decode: make(chan struct{}, 1), + } +} + +func (s *Service) Upload(ctx context.Context, actor uuid.UUID, input UploadInput, body io.Reader) (Attachment, error) { + if err := ValidateFilename(input.Filename); err != nil { + return Attachment{}, err + } + if _, accepted := imageFormats[input.MIMEType]; !accepted { + return Attachment{}, apperr.UnsupportedMedia("only image/png and image/jpeg are accepted") + } + ok, err := s.chats.IsMember(ctx, input.ChatID, actor) + if err != nil { + return Attachment{}, err + } + if !ok { + return Attachment{}, apperr.NotFound("chat not found") + } + if input.SizeBytes > s.config.MaxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if input.SizeBytes <= 0 { + return Attachment{}, apperr.Invalid("a nonempty body with Content-Length is required") + } + if s.blobs == nil { + return Attachment{}, apperr.Unavailable("object storage is unavailable") + } + id := ident.New() + stage := "staging/" + id.String() + discard := true + defer func() { + if !discard { + return + } + cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // A failed cleanup leaves a staging object for the reconciler's grace-based collection. + if err := s.blobs.Delete(cleanup, stage); err != nil { + slog.Warn("staging cleanup deferred", "key", stage, "error", err) + } + }() + hash := sha256.New() + counter := &countReader{r: io.LimitReader(body, s.config.MaxBytes+1)} + err = s.blobs.Put(ctx, stage, io.TeeReader(counter, hash), input.SizeBytes, input.MIMEType) + if err != nil { + _, tooLarge := errors.AsType[*http.MaxBytesError](err) + if counter.n > s.config.MaxBytes || tooLarge { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + return Attachment{}, apperr.Invalid("incomplete upload") + } + return Attachment{}, unavailable(err) + } + var extra [1]byte + n, readErr := counter.Read(extra[:]) + if counter.n > s.config.MaxBytes { + return Attachment{}, apperr.PayloadTooLarge("file exceeds byte limit") + } + if counter.n != input.SizeBytes || n != 0 || !errors.Is(readErr, io.EOF) { + return Attachment{}, apperr.Invalid("body length does not match Content-Length") + } + // Full decoding is bounded by dimensions and one decoder per replica. Body + // transfer itself stays streaming and does not occupy the decoder slot. + select { + case s.decode <- struct{}{}: + case <-ctx.Done(): + return Attachment{}, unavailable(ctx.Err()) + } + err = s.validate(ctx, stage, input.MIMEType) + <-s.decode + if err != nil { + return Attachment{}, err + } + a := Attachment{ + ID: id, + ChatID: input.ChatID, + OwnerID: actor, + Filename: input.Filename, + FileHash: hash.Sum(nil), + SizeBytes: input.SizeBytes, + MIMEType: input.MIMEType, + Status: FileQueued, + UpdatedAt: ident.TimeOf(id), + CreatedAt: ident.TimeOf(id), + } + if s.config.BarrierDir != "" { + base := filepath.Join(s.config.BarrierDir, hex.EncodeToString(a.FileHash)+"-staged") + if err := faultbarrier.Wait(ctx, base, a.ID.String()); err != nil { + return Attachment{}, unavailable(err) + } + } + // COMMIT can succeed even if its reply is lost. From this point only the + // collector, after consulting durable job state, may delete this source. + discard = false + err = s.attachments.Enqueue(ctx, a, stage, func() error { + obj, err := s.blobs.Stat(ctx, stage) + if err != nil { + return unavailable(err) + } + if obj.Size != a.SizeBytes { + return unavailable(errors.New("staging size changed")) + } + return nil + }) + return a, err +} + +func (s *Service) validate(ctx context.Context, stage, mime string) (err error) { + r, err := s.blobs.Open(ctx, stage) + if err != nil { + return unavailable(err) + } + defer func() { + if closeErr := r.Close(); closeErr != nil && err == nil { + err = unavailable(closeErr) + } + }() + tracked := &storageReader{reader: r} + err = ValidateImage(tracked, mime, s.config.MaxPixels) + if tracked.err != nil { + return unavailable(tracked.err) + } + if ctx.Err() != nil { + return unavailable(ctx.Err()) + } + return err +} + +// Image sniffing can replace a reader failure with ErrFormat. Preserve the +// storage failure independently, while ordinary EOF still means malformed bytes. +type storageReader struct { + reader io.Reader + err error +} + +func (r *storageReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil && !errors.Is(err, io.EOF) { + r.err = err + } + return n, err +} + +func (s *Service) Get(ctx context.Context, actor, id uuid.UUID) (Attachment, error) { + return s.attachments.GetForMember(ctx, id, actor) +} + +func (s *Service) Content(ctx context.Context, actor, id uuid.UUID, name string) (Attachment, io.ReadCloser, error) { + a, err := s.Get(ctx, actor, id) + if err != nil { + return Attachment{}, nil, err + } + if a.Status != FileReady { + return Attachment{}, nil, apperr.Conflict("attachment is not ready") + } + key := BlobKey(a.FileHash) + if name != "original" { + found := false + for _, v := range a.Variants { + if name == v.Name { + key, a.MIMEType, a.SizeBytes = v.ObjectKey, v.MIMEType, v.SizeBytes + found = true + break + } + } + if !found { + return Attachment{}, nil, apperr.NotFound("variant not found") + } + } + if s.blobs == nil { + return Attachment{}, nil, apperr.Unavailable("object storage is unavailable") + } + r, err := s.blobs.Open(ctx, key) + if name != "original" && errors.Is(err, ErrObjectMissing) { + return Attachment{}, nil, apperr.NotFound("variant not found") + } + if err != nil { + return Attachment{}, nil, unavailable(err) + } + return a, r, nil +} + +type countReader struct { + r io.Reader + n int64 +} + +func (c *countReader) Read(b []byte) (int, error) { + n, err := c.r.Read(b) + c.n += int64(n) + return n, err +} + +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "object storage is unavailable", err) +} diff --git a/internal/attachment/service_test.go b/internal/attachment/service_test.go new file mode 100644 index 00000000..2872f924 --- /dev/null +++ b/internal/attachment/service_test.go @@ -0,0 +1,181 @@ +package attachment_test + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "messenger/internal/attachment" + "messenger/internal/attachment/store" + chatstore "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" +) + +func TestCollectorRechecksStageAfterConcurrentEnqueue(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + f := reconcileSetup(t, db) + f.exec(`INSERT INTO chat_members VALUES ($1,$2)`, f.chat, f.user) + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + entered := make(chan string, 1) + resume := make(chan struct{}) + var once sync.Once + f.b.stating = func(key string) { + once.Do(func() { + f.b.add(key, data, time.Now().Add(-2*time.Hour)) + entered <- key + select { + case <-resume: + case <-ctx.Done(): + } + }) + } + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), f.b, attachment.Config{}) + uploaded := make(chan error, 1) + go func() { + _, err := s.Upload(ctx, f.user, attachment.UploadInput{ChatID: f.chat, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))}, bytes.NewReader(data)) + uploaded <- err + }() + var key string + select { + case key = <-entered: + case <-ctx.Done(): + t.Fatal("upload never verified staging under lock") + } + r, err := attachment.NewReconciler(db.Pool, f.b, time.Hour) + if err != nil { + t.Fatal(err) + } + collect := make(chan error, 1) + go func() { + stats, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + if err == nil && stats.Repaired != 0 { + err = errors.New("removed concurrent upload") + } + collect <- err + }() + waitReconcileLock(t, ctx, db.Pool) + close(resume) + if err := <-uploaded; err != nil { + t.Fatal(err) + } + if err := <-collect; err != nil { + t.Fatal(err) + } + if !f.b.has(key) { + t.Fatal("accepted source lost") + } +} + +type lostCommitReply struct { + attachment.AttachmentRepository +} + +func (r lostCommitReply) Enqueue(ctx context.Context, a attachment.Attachment, stage string, check func() error) error { + if err := r.AttachmentRepository.Enqueue(ctx, a, stage, check); err != nil { + return err + } + return io.ErrUnexpectedEOF +} + +func TestUploadDeduplicatesContentButKeepsChatScopedIdentities(t *testing.T) { + ctx := context.Background() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + defer db.Close() + actor, c1, c2 := ident.New(), ident.New(), ident.New() + _, err = db.Pool.Exec(ctx, `INSERT INTO users VALUES($1,'alice','Alice','hash',now());`, actor) + if err != nil { + t.Fatal(err) + } + for _, c := range []uuid.UUID{c1, c2} { + _, err = db.Pool.Exec(ctx, `INSERT INTO chats VALUES($1,'room',now())`, c) + if err != nil { + t.Fatal(err) + } + _, err = db.Pool.Exec(ctx, `INSERT INTO chat_members VALUES($1,$2)`, c, actor) + if err != nil { + t.Fatal(err) + } + } + b := newReconcileBlobs() + s := attachment.NewService(store.NewAttachments(db.Pool), chatstore.NewChats(db.Pool), b, attachment.Config{}) + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + var results [10]attachment.Attachment + var wg sync.WaitGroup + for i := range results { + wg.Go(func() { + c := c1 + if i%2 == 1 { + c = c2 + } + var e error + input := attachment.UploadInput{ChatID: c, Filename: "photo.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + results[i], e = s.Upload(ctx, actor, input, bytes.NewReader(data)) + if e != nil { + t.Error(e) + } + }) + } + wg.Wait() + var files, atts, jobs int + err = db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments), + (SELECT count(*) FROM attachment_jobs WHERE state='queued' AND attempt=1 AND published_at IS NULL)`).Scan(&files, &atts, &jobs) + if err != nil || files != 1 || atts != 10 || jobs != 10 || len(b.objects) != 10 { + t.Fatalf("handoff: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err) + } + for _, a := range results { + if a.Status != attachment.FileQueued || !bytes.Equal(b.bodies["staging/"+a.ID.String()], data) { + t.Fatal("missing queued source", a.ID) + } + } + if results[0].ID == results[1].ID { + t.Fatal("deduplication merged chat ACL identity") + } + if _, err := s.Get(ctx, ident.New(), results[0].ID); apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("outsider got %v", err) + } + input := attachment.UploadInput{ChatID: c1, Filename: "lost.png", MIMEType: "image/png", SizeBytes: int64(len(data))} + lost := attachment.NewService(lostCommitReply{store.NewAttachments(db.Pool)}, chatstore.NewChats(db.Pool), b, attachment.Config{}) + if _, err := lost.Upload(ctx, actor, input, bytes.NewReader(data)); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatal("lost commit reply:", err) + } + var source string + if err := db.Pool.QueryRow(ctx, `SELECT j.source_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.filename='lost.png'`).Scan(&source); err != nil || !bytes.Equal(b.bodies[source], data) { + t.Fatal("committed job lost source", err) + } + _, err = db.Pool.Exec(ctx, `CREATE FUNCTION reject_job() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected insert failure'; END $$; + CREATE TRIGGER reject_job BEFORE INSERT ON attachment_jobs FOR EACH ROW EXECUTE FUNCTION reject_job()`) + if err != nil { + t.Fatal(err) + } + other := append(bytes.Clone(data), 0) + input.Filename, input.SizeBytes = "rollback.png", int64(len(other)) + if _, err := s.Upload(ctx, actor, input, bytes.NewReader(other)); err == nil { + t.Fatal("accepted rejected job") + } + if err := db.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM files),(SELECT count(*) FROM attachments),(SELECT count(*) FROM attachment_jobs)`).Scan(&files, &atts, &jobs); err != nil || files != 1 || atts != 11 || jobs != 11 || len(b.objects) != 12 { + t.Fatalf("rollback: files=%d attachments=%d jobs=%d objects=%d err=%v", files, atts, jobs, len(b.objects), err) + } +} diff --git a/internal/attachment/store/postgres.go b/internal/attachment/store/postgres.go new file mode 100644 index 00000000..386340c9 --- /dev/null +++ b/internal/attachment/store/postgres.go @@ -0,0 +1,89 @@ +package store + +import ( + "context" + "crypto/sha256" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" +) + +type Attachments struct{ pool *pgxpool.Pool } + +var _ attachment.AttachmentRepository = (*Attachments)(nil) + +func NewAttachments(pool *pgxpool.Pool) *Attachments { return &Attachments{pool: pool} } + +func (r *Attachments) Enqueue(ctx context.Context, a attachment.Attachment, stage string, checkStage func() error) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + // A request can pause after validation for longer than the GC grace. + // Verify the source after serializing with deletion of this exact key. + stageHash := sha256.Sum256([]byte(stage)) + if err := attachment.LockHash(ctx, tx, stageHash[:]); err != nil { + return err + } + if err := checkStage(); err != nil { + return err + } + if err := attachment.LockHash(ctx, tx, a.FileHash); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + INSERT INTO files(hash,size_bytes,mime_type,status) VALUES ($1,$2,$3,'pending') + ON CONFLICT (hash) DO NOTHING`, a.FileHash, a.SizeBytes, a.MIMEType); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + VALUES ($1,$2,$3,$4,$5,$6)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.FileHash, a.CreatedAt); err != nil { + return err + } + _, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, a.ID, stage) + return err + })) +} + +func (r *Attachments) GetForMember(ctx context.Context, id, actor uuid.UUID) (attachment.Attachment, error) { + var a attachment.Attachment + // A single statement returns one snapshot of the published version, even + // when a worker completes a newer version during a download. + rows, err := r.pool.Query(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.file_hash, f.size_bytes, f.mime_type, + s.status, s.error_code, a.created_at, s.updated_at, + coalesce(v.variant,''),coalesce(v.object_key,''),coalesce(v.mime_type,''), + coalesce(v.size_bytes,0),coalesce(v.width,0),coalesce(v.height,0),v.sha256 + FROM attachments a + JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 + JOIN files f ON f.hash=a.file_hash + JOIN attachment_states s ON s.id=a.id + LEFT JOIN attachment_variants v ON v.attachment_id=a.id AND v.pipeline_version=a.published_version AND s.status='ready' + WHERE a.id=$1 ORDER BY v.variant`, id, actor) + if err != nil { + return a, postgres.Classify(err) + } + defer rows.Close() + for rows.Next() { + var v attachment.Variant + err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.FileHash, &a.SizeBytes, &a.MIMEType, + &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + &v.Name, &v.ObjectKey, &v.MIMEType, &v.SizeBytes, &v.Width, &v.Height, &v.Hash) + if err != nil { + return attachment.Attachment{}, postgres.Classify(err) + } + if v.Name != "" { + a.Variants = append(a.Variants, v) + } + } + if err := rows.Err(); err != nil { + return attachment.Attachment{}, postgres.Classify(err) + } + if a.ID == uuid.Nil { + return a, apperr.NotFound("attachment not found") + } + return a, nil +} diff --git a/internal/attachment/validation.go b/internal/attachment/validation.go new file mode 100644 index 00000000..e74a7f38 --- /dev/null +++ b/internal/attachment/validation.go @@ -0,0 +1,54 @@ +package attachment + +import ( + "bytes" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "strings" + "unicode" + "unicode/utf8" + + "messenger/internal/imaging" + "messenger/internal/platform/apperr" +) + +var imageFormats = map[string]string{ + "image/png": "png", + "image/jpeg": "jpeg", +} + +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return apperr.Invalid("filename must contain 1-255 characters") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return apperr.Invalid("filename contains a forbidden character") + } + } + return nil +} + +// The small header is replayed into the decoder; pixels are allocated only +// after checking dimensions. The caller serializes decoding per replica. +func ValidateImage(r io.Reader, mime string, maxPixels int64) error { + limited := io.LimitReader(r, imaging.MaxUploadBytes+1) + var header bytes.Buffer + cfg, format, err := image.DecodeConfig(io.TeeReader(limited, &header)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + if expectedFormat, accepted := imageFormats[mime]; !accepted || format != expectedFormat { + return apperr.UnsupportedMedia("image does not match Content-Type") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxPixels/int64(cfg.Height) { + return apperr.PayloadTooLarge("image exceeds pixel limit") + } + _, _, err = image.Decode(io.MultiReader(bytes.NewReader(header.Bytes()), limited)) + if err != nil { + return apperr.Wrap(apperr.KindUnsupportedMedia, "invalid image", err) + } + return nil +} diff --git a/internal/attachment/validation_test.go b/internal/attachment/validation_test.go new file mode 100644 index 00000000..0c7b4092 --- /dev/null +++ b/internal/attachment/validation_test.go @@ -0,0 +1,145 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "os" + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" +) + +type chatMember struct { + chatID, userID uuid.UUID +} + +func (m chatMember) IsMember(_ context.Context, chatID, userID uuid.UUID) (bool, error) { + return chatID == m.chatID && userID == m.userID, nil +} + +func TestUploadWithoutStoragePreservesChatAccessCheck(t *testing.T) { + member := chatMember{chatID: uuid.New(), userID: uuid.New()} + s := NewService(nil, member, nil, Config{}) + input := UploadInput{ChatID: member.chatID, Filename: "photo.png", MIMEType: "image/png", SizeBytes: 4} + for _, tc := range []struct { + name string + actor uuid.UUID + kind apperr.Kind + }{ + {name: "member", actor: member.userID, kind: apperr.KindUnavailable}, + {name: "outsider", actor: uuid.New(), kind: apperr.KindNotFound}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := s.Upload(context.Background(), tc.actor, input, strings.NewReader("data")) + if apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want %v", err, tc.kind) + } + }) + } +} + +func TestValidateFilename(t *testing.T) { + for _, name := range []string{"", " ", "../a.png", `a\b.jpg`, "a\x00b", "a\nb", strings.Repeat("я", 256)} { + if err := ValidateFilename(name); apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%q: %v", name, err) + } + } + if err := ValidateFilename("фото 👋.png"); err != nil { + t.Fatal(err) + } +} + +type failingRead struct{ prefix []byte } + +func (r *failingRead) Read(p []byte) (int, error) { + if len(r.prefix) > 0 { + n := copy(p, r.prefix) + r.prefix = r.prefix[n:] + return n, nil + } + return 0, errors.New("injected storage read outage") +} + +type openedBlob struct { + Blobs + reader io.Reader +} + +func (b openedBlob) Open(context.Context, string) (io.ReadCloser, error) { + if reader, ok := b.reader.(io.ReadCloser); ok { + return reader, nil + } + return io.NopCloser(b.reader), nil +} + +type closeErrorBody struct{ io.Reader } + +func (closeErrorBody) Close() error { return errors.New("injected close failure") } + +func TestStagingCloseFailurePreventsPublication(t *testing.T) { + data, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + blobs := openedBlob{reader: closeErrorBody{Reader: bytes.NewReader(data)}} + s := NewService(nil, nil, blobs, Config{}) + if err := s.validate(context.Background(), "staging/test", "image/png"); apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("close failure classified as %v", err) + } +} + +func TestStagingReadOutageIsUnavailableBeforeAndAfterHeader(t *testing.T) { + var pngData bytes.Buffer + if err := png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil { + t.Fatal(err) + } + for _, prefix := range [][]byte{nil, pngData.Bytes()[:33]} { + s := NewService(nil, nil, openedBlob{reader: &failingRead{prefix: prefix}}, Config{}) + err := s.validate(context.Background(), "staging/test", "image/png") + if apperr.KindOf(err) != apperr.KindUnavailable { + t.Fatalf("storage read error classified as %v: %v", apperr.KindOf(err), err) + } + } +} + +func TestValidateImageRejectsCorruptionAndLimitsPixelsBeforeDecode(t *testing.T) { + im := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + im.Set(1, 1, color.NRGBA{R: 200, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, im); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, mime string + data []byte + pixels int64 + kind apperr.Kind + }{ + {"valid", "image/png", b.Bytes(), 6, apperr.KindInternal}, + {"spoof", "image/jpeg", b.Bytes(), 6, apperr.KindUnsupportedMedia}, + {"truncated", "image/png", b.Bytes()[:len(b.Bytes())-20], 6, apperr.KindUnsupportedMedia}, + {"pixels", "image/png", b.Bytes(), 5, apperr.KindPayloadTooLarge}, + {"garbage", "image/png", []byte("not a picture"), 6, apperr.KindUnsupportedMedia}, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateImage(bytes.NewReader(tc.data), tc.mime, tc.pixels) + if tc.name == "valid" { + if err != nil { + t.Fatal(err) + } + return + } + if err == nil || apperr.KindOf(err) != tc.kind { + t.Fatalf("got %v, want kind %v", err, tc.kind) + } + }) + } +} diff --git a/internal/auth/handler.go b/internal/auth/handler.go new file mode 100644 index 00000000..e8c8ccef --- /dev/null +++ b/internal/auth/handler.go @@ -0,0 +1,89 @@ +package auth + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/user" +) + +// Challenges sent with every 401: Basic on the login endpoint, Bearer on +// every protected one. +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +var errCredentialsRequired = apperr.Unauthorized("basic credentials are required") + +type Handler struct { + sessions *Service +} + +func NewHandler(sessions *Service) *Handler { + return &Handler{sessions: sessions} +} + +// Register mounts login on the public router and logout on the router +// guarded by RequireSession. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/auth/sessions", h.login) + protected.DELETE("/auth/sessions/current", h.logout) +} + +type sessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User user.Response `json:"user"` +} + +// login accepts only the Authorization header; the body is not read. +// BasicAuth splits at the first colon, so a password may contain colons. +func (h *Handler) login(c *gin.Context) { + username, password, ok := c.Request.BasicAuth() + if !ok { + unauthorized(c, basicChallenge, errCredentialsRequired) + return + } + issued, err := h.sessions.Login(c.Request.Context(), username, password) + if err != nil { + unauthorized(c, basicChallenge, err) + return + } + httpx.JSON(c, http.StatusCreated, sessionResponse{ + Token: issued.Token, + TokenType: "Bearer", + ExpiresAt: httpx.Timestamp(issued.ExpiresAt), + User: user.ToResponse(issued.User), + }) +} + +func (h *Handler) logout(c *gin.Context) { + token, ok := c.Get(keyToken) + if !ok { + httpx.Error(c, errors.New("logout route is missing the RequireSession middleware")) + return + } + if err := h.sessions.Logout(c.Request.Context(), token.(string)); err != nil { + httpx.Error(c, err) + return + } + // Status alone is buffered by gin; writing the header now sends the 204 + // with an empty body. + c.Status(http.StatusNoContent) + c.Writer.WriteHeaderNow() +} + +// unauthorized adds the challenge only to a 401: an unavailable store is a +// 503 and must not invite the client to retry its credentials. +func unauthorized(c *gin.Context, challenge string, err error) { + if apperr.KindOf(err) == apperr.KindUnauthorized { + c.Header("WWW-Authenticate", challenge) + } + httpx.Error(c, err) +} diff --git a/internal/auth/main_test.go b/internal/auth/main_test.go new file mode 100644 index 00000000..a542ee95 --- /dev/null +++ b/internal/auth/main_test.go @@ -0,0 +1,33 @@ +package auth_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go new file mode 100644 index 00000000..13dbeeae --- /dev/null +++ b/internal/auth/middleware.go @@ -0,0 +1,48 @@ +package auth + +import ( + "strings" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/httpx" +) + +const keyToken = "messenger.session_token" + +// RequireSession admits a request only with Authorization: Bearer of +// a live session and records its owner for the handlers behind it. Any other +// identification, including Basic credentials and X-User-Id, is ignored, so +// it can neither grant access nor change the caller. +func RequireSession(sessions *Service) gin.HandlerFunc { + return func(c *gin.Context) { + token, ok := bearerToken(c.GetHeader("Authorization")) + if !ok { + unauthorized(c, bearerChallenge, ErrUnauthenticated) + return + } + userID, err := sessions.Authenticate(c.Request.Context(), token) + if err != nil { + unauthorized(c, bearerChallenge, err) + return + } + httpx.SetUserID(c, userID) + c.Set(keyToken, token) + c.Next() + } +} + +// bearerToken parses "Bearer ": the scheme is case-insensitive and +// may be followed by several spaces (RFC 7235); the token itself never +// contains whitespace. +func bearerToken(header string) (string, bool) { + scheme, rest, found := strings.Cut(header, " ") + if !found || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token := strings.TrimLeft(rest, " ") + if token == "" || strings.ContainsAny(token, " \t") { + return "", false + } + return token, true +} diff --git a/internal/auth/repository.go b/internal/auth/repository.go new file mode 100644 index 00000000..4a168f59 --- /dev/null +++ b/internal/auth/repository.go @@ -0,0 +1,18 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores the session until its absolute ExpiresAt. + Create(ctx context.Context, s Session) error + // UserID returns the owner of a live session, or ErrUnauthenticated when + // the token is unknown, expired or revoked: the store cannot tell these + // apart, and the client must not either. + UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) + // Delete revokes the session; deleting a missing one is not an error. + Delete(ctx context.Context, tokenHash string) error +} diff --git a/internal/auth/service.go b/internal/auth/service.go new file mode 100644 index 00000000..458a4744 --- /dev/null +++ b/internal/auth/service.go @@ -0,0 +1,64 @@ +package auth + +import ( + "context" + "time" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/user" +) + +var ErrUnauthenticated = apperr.Unauthorized("a valid bearer token is required") + +// Credentials is what this module needs from the users module: the user +// whose password matches, or user.ErrInvalidCredentials. +type Credentials interface { + Authenticate(ctx context.Context, username, password string) (user.User, error) +} + +// Issued is a new session as the client sees it; Token is shown only here. +type Issued struct { + Token string + ExpiresAt time.Time + User user.User +} + +type Service struct { + users Credentials + sessions Repository + ttl time.Duration +} + +func NewService(users Credentials, sessions Repository, ttl time.Duration) *Service { + return &Service{users: users, sessions: sessions, ttl: ttl} +} + +// Login checks the credentials first, so a store outage never hands out a +// token without a stored session. The expiry is fixed here and never +// extended; it is truncated to milliseconds, the precision of both the API +// timestamps and the store. +func (s *Service) Login(ctx context.Context, username, password string) (Issued, error) { + u, err := s.users.Authenticate(ctx, username, password) + if err != nil { + return Issued{}, err + } + token := newToken() + expiresAt := time.Now().Add(s.ttl).Truncate(time.Millisecond).UTC() + if err := s.sessions.Create(ctx, Session{TokenHash: HashToken(token), UserID: u.ID, ExpiresAt: expiresAt}); err != nil { + return Issued{}, err + } + return Issued{Token: token, ExpiresAt: expiresAt, User: u}, nil +} + +// Authenticate returns the owner of the token's live session or +// ErrUnauthenticated. A store failure stays an unavailability, never +// ErrUnauthenticated: an outage must not look like a bad token to the client. +func (s *Service) Authenticate(ctx context.Context, token string) (uuid.UUID, error) { + return s.sessions.UserID(ctx, HashToken(token)) +} + +func (s *Service) Logout(ctx context.Context, token string) error { + return s.sessions.Delete(ctx, HashToken(token)) +} diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go new file mode 100644 index 00000000..e3cd3fb2 --- /dev/null +++ b/internal/auth/service_test.go @@ -0,0 +1,156 @@ +package auth_test + +import ( + "context" + "encoding/base64" + "errors" + "strings" + "testing" + "time" + + "messenger/internal/auth" + authstore "messenger/internal/auth/store" + "messenger/internal/platform/apperr" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +type fixture struct { + ctx context.Context + svc *auth.Service + alice user.User +} + +// newFixture wires the real users service and session store over empty +// databases and registers alice. +func newFixture(t *testing.T, ttl time.Duration) fixture { + t.Helper() + db.Reset(t) + kv.Reset(t) + ctx := context.Background() + users := user.NewService(userstore.NewUsers(db.Pool)) + alice, err := users.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatal(err) + } + return fixture{ctx: ctx, svc: auth.NewService(users, authstore.NewSessions(kv.Client), ttl), alice: alice} +} + +func TestLoginIssuesOpaqueTokenBackedByStoredSession(t *testing.T) { + f := newFixture(t, time.Hour) + before := time.Now() + issued, err := f.svc.Login(f.ctx, "alice", pw) + after := time.Now() + if err != nil { + t.Fatalf("Login: %v", err) + } + raw, err := base64.RawURLEncoding.DecodeString(issued.Token) + if err != nil || len(raw) != 32 { + t.Fatalf("token %q must be 32 random bytes in base64url, got %d bytes, %v", issued.Token, len(raw), err) + } + if strings.Contains(issued.Token, f.alice.ID.String()) || strings.Contains(issued.Token, "alice") { + t.Fatal("token must not carry the identity") + } + if issued.User != f.alice { + t.Fatalf("User = %+v, want alice", issued.User) + } + if issued.ExpiresAt.Before(before.Add(time.Hour).Add(-time.Millisecond)) || issued.ExpiresAt.After(after.Add(time.Hour)) { + t.Fatalf("ExpiresAt = %v, want login time + 1h", issued.ExpiresAt) + } + + key := "session:" + auth.HashToken(issued.Token) + owner, err := kv.Client.Get(f.ctx, key).Result() + if err != nil || owner != f.alice.ID.String() { + t.Fatalf("stored owner = %q, %v; want alice", owner, err) + } + expireAt, err := kv.Client.PExpireTime(f.ctx, key).Result() + if err != nil || expireAt != time.Duration(issued.ExpiresAt.UnixMilli())*time.Millisecond { + t.Fatalf("store expiry = %v, %v; want the absolute ExpiresAt %d ms", expireAt, err, issued.ExpiresAt.UnixMilli()) + } + keys, err := kv.Client.Keys(f.ctx, "*").Result() + if err != nil { + t.Fatal(err) + } + for _, k := range keys { + if strings.Contains(k, issued.Token) { + t.Fatal("the raw token must never be stored") + } + } +} + +func TestLoginWithBadCredentialsCreatesNoSession(t *testing.T) { + f := newFixture(t, time.Hour) + for name, in := range map[string][2]string{ + "wrong password": {"alice", pw + "!"}, + "unknown user": {"mallory", pw}, + } { + if _, err := f.svc.Login(f.ctx, in[0], in[1]); !errors.Is(err, user.ErrInvalidCredentials) { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if n, err := kv.Client.DBSize(f.ctx).Result(); err != nil || n != 0 { + t.Fatalf("sessions = %d, %v; want none", n, err) + } +} + +func TestAuthenticateResolvesTheOwner(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if first.Token == second.Token { + t.Fatal("every login must get its own token") + } + for _, token := range []string{first.Token, second.Token} { + id, err := f.svc.Authenticate(f.ctx, token) + if err != nil || id != f.alice.ID { + t.Fatalf("Authenticate = %v, %v; want alice", id, err) + } + } + _, err := f.svc.Authenticate(f.ctx, "unknown-token") + if !errors.Is(err, auth.ErrUnauthenticated) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Fatalf("unknown token: err = %v, want ErrUnauthenticated", err) + } +} + +func TestLogoutRevokesOnlyThatSession(t *testing.T) { + f := newFixture(t, time.Hour) + first, _ := f.svc.Login(f.ctx, "alice", pw) + second, _ := f.svc.Login(f.ctx, "alice", pw) + if err := f.svc.Logout(f.ctx, first.Token); err != nil { + t.Fatalf("Logout: %v", err) + } + if _, err := f.svc.Authenticate(f.ctx, first.Token); !errors.Is(err, auth.ErrUnauthenticated) { + t.Fatalf("revoked token: err = %v, want ErrUnauthenticated", err) + } + if id, err := f.svc.Authenticate(f.ctx, second.Token); err != nil || id != f.alice.ID { + t.Fatalf("other session: %v, %v; want still valid", id, err) + } +} + +// Valkey enforces the expiry by the Docker VM clock, which may lag this host +// by a fraction of a second, so the expiry is awaited with a tolerance. +func TestSessionExpiresAfterTTL(t *testing.T) { + const clockTolerance = 3 * time.Second + f := newFixture(t, time.Second) + issued, err := f.svc.Login(f.ctx, "alice", pw) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil { + t.Fatalf("fresh session: %v", err) + } + time.Sleep(time.Until(issued.ExpiresAt)) + deadline := time.Now().Add(clockTolerance) + for { + _, err := f.svc.Authenticate(f.ctx, issued.Token) + if errors.Is(err, auth.ErrUnauthenticated) { + return + } + if time.Now().After(deadline) { + t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance) + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/internal/auth/session.go b/internal/auth/session.go new file mode 100644 index 00000000..daedb116 --- /dev/null +++ b/internal/auth/session.go @@ -0,0 +1,39 @@ +// Package auth authenticates callers. +// +// A login with Basic credentials creates a server-side session and returns an +// opaque bearer token; every protected request presents the token, and logout +// deletes the session, which revokes the token at once. The store keeps only +// the SHA-256 of a token: a token is a random 256-bit value, so a fast digest +// cannot be reversed by guessing, and a leaked store yields no usable tokens. +// Passwords need the opposite treatment (a slow KDF), see platform/password. +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "time" + + "github.com/google/uuid" +) + +type Session struct { + TokenHash string + UserID uuid.UUID + ExpiresAt time.Time +} + +// newToken returns 32 random bytes in base64url: 43 characters, 256 bits. +func newToken() string { + raw := make([]byte, 32) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(raw) + return base64.RawURLEncoding.EncodeToString(raw) +} + +// HashToken is the key under which a token's session is stored. +func HashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/auth/store/valkey.go b/internal/auth/store/valkey.go new file mode 100644 index 00000000..ad7a36e1 --- /dev/null +++ b/internal/auth/store/valkey.go @@ -0,0 +1,64 @@ +// Package store holds the session store implementations. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "messenger/internal/auth" + "messenger/internal/platform/apperr" +) + +const keyPrefix = "session:" + +// Sessions keeps one key per session, session:, holding the +// owner's id. Valkey expires the key itself, so no cleanup job is needed. +type Sessions struct { + client *redis.Client +} + +var _ auth.Repository = (*Sessions)(nil) + +func NewSessions(client *redis.Client) *Sessions { + return &Sessions{client: client} +} + +// Create sets an absolute expiry (EXAT/PXAT): a restart of the store replays +// it from the append-only file and does not extend the session. +func (r *Sessions) Create(ctx context.Context, s auth.Session) error { + err := r.client.SetArgs(ctx, keyPrefix+s.TokenHash, s.UserID.String(), redis.SetArgs{ExpireAt: s.ExpiresAt}).Err() + if err != nil { + return unavailable(err) + } + return nil +} + +func (r *Sessions) UserID(ctx context.Context, tokenHash string) (uuid.UUID, error) { + raw, err := r.client.Get(ctx, keyPrefix+tokenHash).Result() + if errors.Is(err, redis.Nil) { + return uuid.Nil, auth.ErrUnauthenticated + } + if err != nil { + return uuid.Nil, unavailable(err) + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, fmt.Errorf("session holds a malformed user id: %w", err) + } + return id, nil +} + +func (r *Sessions) Delete(ctx context.Context, tokenHash string) error { + if err := r.client.Del(ctx, keyPrefix+tokenHash).Err(); err != nil { + return unavailable(err) + } + return nil +} + +func unavailable(err error) error { + return apperr.Wrap(apperr.KindUnavailable, "session store unavailable", err) +} diff --git a/internal/broker/rabbit.go b/internal/broker/rabbit.go new file mode 100644 index 00000000..e4cf9f38 --- /dev/null +++ b/internal/broker/rabbit.go @@ -0,0 +1,115 @@ +// Package broker owns the AMQP connection for one publisher or consumer. +package broker + +import ( + "context" + "errors" + "net" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const queue = "attachment.jobs" + +type link struct { + conn *amqp.Connection + channel *amqp.Channel +} + +func open(ctx context.Context, url string) (*link, error) { + conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 5 * time.Second, Dial: func(network, addr string) (net.Conn, error) { + d := net.Dialer{Timeout: 3 * time.Second} + c, err := d.DialContext(ctx, network, addr) + if err == nil { + err = c.SetDeadline(time.Now().Add(5 * time.Second)) + } + return c, err + }}) + // Parse errors can contain the complete connection URL. + if err != nil { + return nil, errors.New("RabbitMQ connection unavailable") + } + l := &link{conn: conn} + l.channel, err = conn.Channel() + if err == nil { + // PostgreSQL limits processing attempts. A broker redelivery can occur + // before PostgreSQL is reachable and must leave the job recoverable. + _, err = l.channel.QueueDeclare(queue, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", "x-delivery-limit": int32(-1), + }) + } + if err != nil { + l.Close() + return nil, err + } + return l, nil +} + +func (l *link) Close() { _ = l.conn.CloseDeadline(time.Now().Add(time.Second)) } + +type Publisher struct { + *link + returned <-chan amqp.Return +} + +func NewPublisher(ctx context.Context, url string) (*Publisher, error) { + l, err := open(ctx, url) + if err != nil { + return nil, err + } + if err = l.channel.Confirm(false); err != nil { + l.Close() + return nil, err + } + return &Publisher{l, l.channel.NotifyReturn(make(chan amqp.Return, 1))}, nil +} + +// Publish has one in-flight confirm on its channel. An ambiguous result means +// the caller discards this publisher and leaves the outbox row pending. +func (p *Publisher) Publish(ctx context.Context, body []byte) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + stop := context.AfterFunc(ctx, p.Close) + defer stop() + confirmation, err := p.channel.PublishWithDeferredConfirmWithContext(ctx, "", queue, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}) + if err != nil { + return err + } + ok, err := confirmation.WaitContext(ctx) + if err != nil { + return err + } + if !ok { + return errors.New("broker did not confirm publication") + } + // RabbitMQ sends basic.return before the associated publisher confirm. + select { + case <-p.returned: + return errors.New("broker returned publication") + default: + return nil + } +} + +type Consumer struct { + *link + Deliveries <-chan amqp.Delivery +} + +func NewConsumer(ctx context.Context, url string) (*Consumer, error) { + l, err := open(ctx, url) + if err != nil { + return nil, err + } + if err = l.channel.Qos(1, 0, false); err != nil { + l.Close() + return nil, err + } + d, err := l.channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil) + if err != nil { + l.Close() + return nil, err + } + return &Consumer{l, d}, nil +} diff --git a/internal/broker/rabbit_test.go b/internal/broker/rabbit_test.go new file mode 100644 index 00000000..7143d9d8 --- /dev/null +++ b/internal/broker/rabbit_test.go @@ -0,0 +1,56 @@ +package broker + +import ( + "context" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func TestConfirmAndRedeliverUnacked(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + c, err := testcontainers.Run(ctx, "rabbitmq:4.1.4-alpine", testcontainers.WithEnv(map[string]string{"RABBITMQ_DEFAULT_USER": "lab", "RABBITMQ_DEFAULT_PASS": "test-only"}), testcontainers.WithExposedPorts("5672/tcp"), testcontainers.WithWaitStrategy(wait.ForLog("Server startup complete"))) + if err != nil { + t.Fatal(err) + } + testcontainers.CleanupContainer(t, c) + endpoint, err := c.PortEndpoint(ctx, "5672/tcp", "") + if err != nil { + t.Fatal(err) + } + url := "amqp://lab:test-only@" + endpoint + "/" + pub, err := NewPublisher(ctx, url) + if err != nil { + t.Fatal(err) + } + defer pub.Close() + if err = pub.Publish(ctx, []byte("work")); err != nil { + t.Fatal(err) + } + // Broker redeliveries while PostgreSQL is unreachable do not consume + // the application budget. Exercise more than the quorum queue default. + for attempt := 0; attempt <= 22; attempt++ { + consumer, err := NewConsumer(ctx, url) + if err != nil { + t.Fatal(err) + } + select { + case msg := <-consumer.Deliveries: + if string(msg.Body) != "work" || msg.Redelivered != (attempt > 0) { + t.Fatalf("unexpected delivery on attempt %d", attempt) + } + if attempt == 22 { + if err := msg.Ack(false); err != nil { + t.Fatal(err) + } + } + case <-time.After(5 * time.Second): + consumer.Close() + t.Fatalf("confirmed job missing on delivery %d", attempt+1) + } + consumer.Close() + } +} diff --git a/internal/chat/attachment_handler_test.go b/internal/chat/attachment_handler_test.go new file mode 100644 index 00000000..764432f5 --- /dev/null +++ b/internal/chat/attachment_handler_test.go @@ -0,0 +1,72 @@ +package chat_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/httpx" +) + +func messageRouter(f fixture, actor uuid.UUID) *gin.Engine { + r := gin.New() + r.Use(func(c *gin.Context) { httpx.SetUserID(c, actor) }) + chat.NewHandler(f.svc).Register(r) + return r +} + +func sendJSON(r http.Handler, chatID uuid.UUID, key, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/chats/"+chatID.String()+"/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", key) + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + return rec +} + +func TestMessageAttachmentJSONEmptyArrayAndOmitted(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for i, body := range []string{`{"text":"hello","attachment_ids":[]}`, `{"text":"hello"}`} { + rec := sendJSON(r, c, fmt.Sprint(i), body) + if rec.Code != http.StatusCreated { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &fields); err != nil { + t.Fatal(err) + } + if string(fields["attachment_ids"]) != "[]" { + t.Fatalf("attachment_ids = %s, want []", fields["attachment_ids"]) + } + } +} + +func TestMessageAttachmentJSONRejectsInvalidArrays(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + id := uuid.NewString() + for _, raw := range []string{ + "null", `"not an array"`, "{}", "1", "[null]", "[1]", `["bad"]`, + `["00000000-0000-0000-0000-000000000000"]`, + fmt.Sprintf("[%q]", strings.ReplaceAll(id, "-", "")), + fmt.Sprintf("[%q,%q]", id, strings.ToUpper(id)), + "[" + strings.TrimSuffix(strings.Repeat(fmt.Sprintf("%q,", id), 11), ",") + "]", + } { + t.Run(raw, func(t *testing.T) { + rec := sendJSON(r, c, "invalid", `{"text":"hello","attachment_ids":`+raw+"}") + if rec.Code != http.StatusBadRequest { + t.Fatalf("send: %d %s", rec.Code, rec.Body) + } + }) + } +} diff --git a/internal/chat/attachment_http_test.go b/internal/chat/attachment_http_test.go new file mode 100644 index 00000000..3850e3ba --- /dev/null +++ b/internal/chat/attachment_http_test.go @@ -0,0 +1,163 @@ +package chat_test + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, hash[:], "pending", id, chatID, owner) + if err != nil { + t.Fatal(err) + } + _, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code,lease_token,lease_until) + VALUES($1,$2,$3,CASE WHEN $3='failed' THEN 'processing_failed' END,CASE WHEN $3='processing' THEN gen_random_uuid() END, + CASE WHEN $3='processing' THEN now()+interval '1 minute' END)`, id, "staging/"+id.String(), status) + if err != nil { + t.Fatal(err) + } + return id +} + +func TestAttachmentHTTPRoundTripReplayAndOrderedConflict(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + a, b := f.attachment(t, c, f.bob, "queued"), f.attachment(t, c, f.alice, "queued") + r := messageRouter(f, f.alice) + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, b, a) + first := sendJSON(r, c, "attachments", body) + if first.Code != http.StatusCreated { + t.Fatalf("attachment-only send: %d %s", first.Code, first.Body) + } + var message chat.MessageResponse + if err := json.Unmarshal(first.Body.Bytes(), &message); err != nil { + t.Fatal(err) + } + if message.Text != "" || !reflect.DeepEqual(message.AttachmentIDs, []string{b.String(), a.String()}) { + t.Fatalf("message = %+v", message) + } + if again := sendJSON(r, c, "attachments", body); again.Code != http.StatusCreated || again.Body.String() != first.Body.String() { + t.Fatalf("replay: %d %s, want %s", again.Code, again.Body, first.Body) + } + for _, changed := range []string{ + fmt.Sprintf(`{"text":"","attachment_ids":[%q,%q]}`, a, b), + fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, b), + fmt.Sprintf(`{"text":"different","attachment_ids":[%q,%q]}`, b, a), + } { + if rec := sendJSON(r, c, "attachments", changed); rec.Code != http.StatusConflict { + t.Fatalf("changed payload: %d %s", rec.Code, rec.Body) + } + } + rec := httptest.NewRecorder() + r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/chats/"+c.String()+"/messages", nil)) + var page pagination.PageResponse[chat.MessageResponse] + if rec.Code != http.StatusOK { + t.Fatalf("history: %d %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], message) { + t.Fatalf("history = %+v, want %+v", page, message) + } + without := sendJSON(r, c, "empty", `{"text":"text-only"}`) + empty := sendJSON(r, c, "empty", `{"attachment_ids":[],"text":"text-only"}`) + if without.Code != http.StatusCreated || empty.Code != http.StatusCreated || without.Body.String() != empty.Body.String() { + t.Fatalf("omitted vs empty replay: %d %s; %d %s", without.Code, without.Body, empty.Code, empty.Body) + } +} + +func TestHTTPReplayAfterAttachmentFailsAndMembershipChanges(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + a := f.attachment(t, c, f.alice, "queued") + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, a) + router := messageRouter(f, f.alice) + first := sendJSON(router, c, "key", body) + if first.Code != http.StatusCreated { + t.Fatal(first.Code, first.Body) + } + if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state='failed',error_code='source_missing' WHERE attachment_id=$1`, a); err != nil { + t.Fatal(err) + } + replay := sendJSON(router, c, "key", body) + if replay.Code != http.StatusCreated || replay.Body.String() != first.Body.String() { + t.Fatalf("replay=%d %s, want original %s", replay.Code, replay.Body, first.Body) + } + changed := fmt.Sprintf(`{"text":"changed","attachment_ids":[%q]}`, ident.New()) + if r := sendJSON(router, c, "key", changed); r.Code != http.StatusConflict { + t.Fatalf("changed body: %d %s", r.Code, r.Body) + } + if r := sendJSON(router, c, "new", body); r.Code != http.StatusConflict { + t.Fatalf("new message: %d %s", r.Code, r.Body) + } + if _, err := db.Pool.Exec(f.ctx, `DELETE FROM chat_members WHERE chat_id=$1 AND user_id=$2`, c, f.alice); err != nil { + t.Fatal(err) + } + if r := sendJSON(router, c, "key", body); r.Code != http.StatusNotFound { + t.Fatalf("former member: %d %s", r.Code, r.Body) + } +} + +func TestAttachmentHTTPStatusAndChatAccess(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + other := f.newChat(t, f.alice) + ready := f.attachment(t, c, f.bob, "queued") + foreign := f.attachment(t, other, f.alice, "queued") + failed := f.attachment(t, c, f.alice, "failed") + pending := f.attachment(t, c, f.alice, "processing") + for _, tc := range []struct { + name string + actor, attachment uuid.UUID + want int + }{ + {"unknown", f.alice, ident.New(), http.StatusNotFound}, + {"foreign", f.alice, foreign, http.StatusNotFound}, + {"failed", f.alice, failed, http.StatusConflict}, + {"processing", f.alice, pending, http.StatusCreated}, + {"outsider", f.eve, ready, http.StatusNotFound}, + {"other-owner", f.alice, ready, http.StatusCreated}, + } { + t.Run(tc.name, func(t *testing.T) { + body := fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, tc.attachment) + rec := sendJSON(messageRouter(f, tc.actor), c, tc.name, body) + if rec.Code != tc.want { + t.Fatalf("send: %d %s, want %d", rec.Code, rec.Body, tc.want) + } + }) + } +} + +func TestMessageJSONRequiresTextString(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + r := messageRouter(f, f.alice) + for _, body := range []string{"null", `{}`, `{"text":null}`, `{"text":1}`, `{"text":[]}`, `{"text":"","attachment_ids":[]}`} { + if rec := sendJSON(r, c, "text", body); rec.Code != http.StatusBadRequest { + t.Fatalf("send %s: %d %s", body, rec.Code, rec.Body) + } + } + if rec := sendJSON(r, c, "text", `{"text":"valid"}`); rec.Code != http.StatusCreated { + t.Fatalf("valid payload after rejected requests: %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/chat/attachment_message_test.go b/internal/chat/attachment_message_test.go new file mode 100644 index 00000000..d14678f8 --- /dev/null +++ b/internal/chat/attachment_message_test.go @@ -0,0 +1,66 @@ +package chat_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestAttachmentMessageValidatesAndOwnsOrderedIDs(t *testing.T) { + a, b := ident.New(), ident.New() + ids := []uuid.UUID{b, a} + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), "", ids...) + if err != nil || m.Text != "" || !reflect.DeepEqual(m.AttachmentIDs, ids) { + t.Fatalf("attachment-only message = %+v, %v", m, err) + } + ids[0] = a + if m.AttachmentIDs[0] != b { + t.Fatal("message aliases caller's attachment slice") + } + for _, tc := range []struct { + text string + ids []uuid.UUID + }{ + {"ok", []uuid.UUID{a, a}}, + {"ok", []uuid.UUID{uuid.Nil}}, + {"ok", make([]uuid.UUID, 11)}, + {strings.Repeat("x", 4001), []uuid.UUID{a}}, + {"nul\x00", []uuid.UUID{a}}, + } { + _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), tc.text, tc.ids...) + if err == nil || apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("invalid message accepted: %v", err) + } + } + ten := make([]uuid.UUID, 10) + for i := range ten { + ten[i] = ident.New() + } + if _, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " \n", ten...); err != nil { + t.Fatalf("ten attachments: %v", err) + } +} + +func TestSameContentIncludesOrderedAttachmentsAndTreatsNilAsEmpty(t *testing.T) { + a, b := ident.New(), ident.New() + m := chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{b, a}} + for _, tc := range []struct { + ids []uuid.UUID + equal bool + }{ + {[]uuid.UUID{b, a}, true}, {[]uuid.UUID{a, b}, false}, {[]uuid.UUID{a}, false}, {nil, false}, + } { + if got := m.SameContent(chat.Message{Text: "hello", AttachmentIDs: tc.ids}); got != tc.equal { + t.Fatalf("attachments %v: equality=%v, want %v", tc.ids, got, tc.equal) + } + } + if !(chat.Message{Text: "hello"}).SameContent(chat.Message{Text: "hello", AttachmentIDs: []uuid.UUID{}}) { + t.Fatal("nil attachments must equal empty attachments") + } +} diff --git a/internal/chat/chat.go b/internal/chat/chat.go new file mode 100644 index 00000000..e84ff665 --- /dev/null +++ b/internal/chat/chat.go @@ -0,0 +1,86 @@ +// Package chat is the chats module. Entities mirror table rows and +// carry no behaviour; membership is a query rather than a field, and +// ChatWithMembers is the projection the API renders. Both entities derive +// CreatedAt from their UUIDv7 id, so the history order (created_at, id) is +// plain id order. +package chat + +import ( + "bytes" + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxTitleCodePoints = 100 + MaxMembers = 100 +) + +// ErrNotFound covers both a missing chat and a chat the caller is not a +// member of, so outsiders cannot probe for existence. +var ErrNotFound = apperr.NotFound("chat not found") + +// ErrKeyReused answers a send whose Idempotency-Key is already bound to a +// message with a different text. +var ErrKeyReused = apperr.Conflict("Idempotency-Key was already used with a different message") + +type Chat struct { + ID uuid.UUID + Title string + CreatedAt time.Time +} + +type ChatWithMembers struct { + Chat + MemberIDs []uuid.UUID +} + +func NewChat(id uuid.UUID, title string) (Chat, error) { + if strings.TrimSpace(title) == "" { + return Chat{}, apperr.Invalid("title must not be blank") + } + if utf8.RuneCountInString(title) > MaxTitleCodePoints { + return Chat{}, apperr.Invalid(fmt.Sprintf("title must be at most %d characters", MaxTitleCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(title, 0) { + return Chat{}, apperr.Invalid("title must not contain the NUL character") + } + return Chat{ID: id, Title: title, CreatedAt: ident.TimeOf(id)}, nil +} + +// ValidateMembers checks the member list of a new chat and returns its own +// copy ordered by id: members are a set, and this is the one order in which +// every read returns them. +func ValidateMembers(memberIDs []uuid.UUID, creator uuid.UUID) ([]uuid.UUID, error) { + if len(memberIDs) == 0 { + return nil, apperr.Invalid("member_ids must not be empty") + } + if len(memberIDs) > MaxMembers { + return nil, apperr.Invalid(fmt.Sprintf("member_ids must have at most %d entries", MaxMembers)) + } + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + for _, m := range memberIDs { + if m == uuid.Nil { + return nil, apperr.Invalid("member_ids must not contain the nil uuid") + } + if _, dup := seen[m]; dup { + return nil, apperr.Invalid("member_ids must be unique") + } + seen[m] = struct{}{} + } + if _, ok := seen[creator]; !ok { + return nil, apperr.Invalid("member_ids must include the creator") + } + members := slices.Clone(memberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { return bytes.Compare(a[:], b[:]) }) + return members, nil +} diff --git a/internal/chat/chat_test.go b/internal/chat/chat_test.go new file mode 100644 index 00000000..f63f60e3 --- /dev/null +++ b/internal/chat/chat_test.go @@ -0,0 +1,88 @@ +package chat_test + +import ( + "strings" + "testing" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewChatDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + c, err := chat.NewChat(id, "Проект") + if err != nil { + t.Fatalf("NewChat error: %v", err) + } + if c.ID != id || c.Title != "Проект" || !c.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("unexpected chat %+v", c) + } +} + +func TestNewChatValidatesTitle(t *testing.T) { + // PostgreSQL text cannot hold U+0000, so a title with it is invalid input, + // not a storage failure. + for _, title := range []string{"", " ", strings.Repeat("я", 101), "nul\x00inside"} { + _, err := chat.NewChat(ident.New(), title) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("title %q must be rejected as invalid, got %v", title, err) + } + } + if _, err := chat.NewChat(ident.New(), strings.Repeat("я", 100)); err != nil { + t.Errorf("100 code points must be accepted: %v", err) + } +} + +// Members come back in one canonical order (by id) whatever order the client +// sent, so the created chat and every later read of it agree. +func TestValidateMembersCopiesInCanonicalOrder(t *testing.T) { + alice, bob := ident.New(), ident.New() + input := []uuid.UUID{bob, alice} + members, err := chat.ValidateMembers(input, alice) + if err != nil { + t.Fatalf("ValidateMembers error: %v", err) + } + if len(members) != 2 || members[0] != alice || members[1] != bob { + t.Fatalf("members must be ordered by id, got %v", members) + } + input[1] = ident.New() + if members[0] != alice { + t.Fatal("result must not alias the input slice") + } +} + +func TestValidateMembersRejectsInvalidLists(t *testing.T) { + alice, bob := ident.New(), ident.New() + cases := map[string][]uuid.UUID{ + "empty": {}, + "nil": nil, + "missing creator": {bob}, + "duplicate": {alice, bob, bob}, + "duplicate self": {alice, alice}, + "too many": append([]uuid.UUID{alice}, many(100)...), + "nil uuid member": {alice, uuid.Nil}, + } + for name, members := range cases { + _, err := chat.ValidateMembers(members, alice) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: want invalid, got %v", name, err) + } + if err != nil && !strings.Contains(err.Error(), "member_ids") { + t.Errorf("%s: message must name the field: %v", name, err) + } + } + if _, err := chat.ValidateMembers(append([]uuid.UUID{alice}, many(99)...), alice); err != nil { + t.Errorf("100 members must be accepted: %v", err) + } +} + +func many(n int) []uuid.UUID { + out := make([]uuid.UUID, n) + for i := range out { + out[i] = ident.New() + } + return out +} diff --git a/internal/chat/cleanup.go b/internal/chat/cleanup.go new file mode 100644 index 00000000..13db162e --- /dev/null +++ b/internal/chat/cleanup.go @@ -0,0 +1,30 @@ +package chat + +import ( + "context" + "log/slog" + "time" +) + +// RunKeyCleanup purges expired Idempotency-Keys every period until ctx is done. +func RunKeyCleanup(ctx context.Context, log *slog.Logger, svc *Service, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + deleted, err := svc.PurgeExpiredKeys(ctx) + if err != nil { + if ctx.Err() == nil { + log.Warn("idempotency key cleanup failed", "deleted", deleted, "error", err.Error()) + } + continue + } + if deleted > 0 { + log.Info("idempotency keys purged", "deleted", deleted) + } + } +} diff --git a/internal/chat/handler.go b/internal/chat/handler.go new file mode 100644 index 00000000..95dc9089 --- /dev/null +++ b/internal/chat/handler.go @@ -0,0 +1,253 @@ +package chat + +import ( + "bytes" + "encoding/json" + "net/http" + "regexp" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +type Handler struct { + chats *Service +} + +func NewHandler(chats *Service) *Handler { + return &Handler{chats: chats} +} + +// Register mounts the chat endpoints on a router guarded by the authentication middleware. +func (h *Handler) Register(protected gin.IRouter) { + protected.POST("/chats", httpx.WithUser(h.create)) + protected.GET("/chats", httpx.WithUser(h.list)) + protected.GET("/chats/:chat_id", httpx.WithUser(h.get)) + protected.POST("/chats/:chat_id/messages", httpx.WithUser(h.send)) + protected.GET("/chats/:chat_id/messages", httpx.WithUser(h.history)) +} + +type createChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type createMessageRequest struct { + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` +} + +type ChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type MessageResponse struct { + AttachmentIDs []string `json:"attachment_ids"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` +} + +func toChatResponse(c ChatWithMembers) ChatResponse { + members := make([]string, 0, len(c.MemberIDs)) + for _, m := range c.MemberIDs { + members = append(members, m.String()) + } + return ChatResponse{ + ID: c.ID.String(), + Title: c.Title, + MemberIDs: members, + CreatedAt: httpx.Timestamp(c.CreatedAt), + } +} + +func toMessageResponse(m Message) MessageResponse { + ids := make([]string, 0, len(m.AttachmentIDs)) + for _, id := range m.AttachmentIDs { + ids = append(ids, id.String()) + } + return MessageResponse{ + AttachmentIDs: ids, + ID: m.ID.String(), + ChatID: m.ChatID.String(), + SenderID: m.SenderID.String(), + Text: m.Text, + CreatedAt: httpx.Timestamp(m.CreatedAt), + } +} + +func (h *Handler) create(c *gin.Context, actor uuid.UUID) { + var req createChatRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + members := make([]uuid.UUID, 0, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + id, err := ident.Parse(raw) + if err != nil { + httpx.Error(c, apperr.Invalid("member_ids must contain valid UUIDs")) + return + } + members = append(members, id) + } + chat, err := h.chats.Create(c.Request.Context(), actor, req.Title, members) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toChatResponse(chat)) +} + +func (h *Handler) list(c *gin.Context, actor uuid.UUID) { + limit, after, err := listing(c, actor) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.List(c.Request.Context(), actor, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, actor, toChatResponse)) +} + +func (h *Handler) get(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + chat, err := h.chats.Get(c.Request.Context(), actor, chatID) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, toChatResponse(chat)) +} + +func (h *Handler) send(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + key, err := idempotencyKey(c) + if err != nil { + httpx.Error(c, err) + return + } + var req createMessageRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + if req.Text == nil { + httpx.Error(c, apperr.Invalid("text must be a string")) + return + } + attachments, err := decodeAttachmentIDs(req.AttachmentIDs) + if err != nil { + httpx.Error(c, err) + return + } + message, err := h.chats.Send(c.Request.Context(), actor, MessagePayload{ChatID: chatID, IdempotencyKey: key, Text: *req.Text, AttachmentIDs: attachments}) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, toMessageResponse(message)) +} + +func (h *Handler) history(c *gin.Context, actor uuid.UUID) { + chatID, err := chatParam(c) + if err != nil { + httpx.Error(c, err) + return + } + limit, after, err := listing(c, chatID) + if err != nil { + httpx.Error(c, err) + return + } + page, err := h.chats.History(c.Request.Context(), actor, chatID, after, limit) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, pagination.ToResponse(page, chatID, toMessageResponse)) +} + +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +// idempotencyKey rejects two values of the header as ambiguous. +func idempotencyKey(c *gin.Context) (string, error) { + values := c.Request.Header.Values("Idempotency-Key") + if len(values) != 1 || !keyPattern.MatchString(values[0]) { + return "", apperr.Invalid("Idempotency-Key header is required: one value of 1-128 characters A-Z a-z 0-9 . _ : -") + } + return values[0], nil +} + +func chatParam(c *gin.Context) (uuid.UUID, error) { + id, err := ident.Parse(c.Param("chat_id")) + if err != nil { + return uuid.Nil, apperr.Invalid("chat_id must be a UUID") + } + return id, nil +} + +// listing parses limit and cursor; the cursor must belong to scope, and a +// present but empty cursor is rejected like a malformed one. +func listing(c *gin.Context, scope uuid.UUID) (int, *uuid.UUID, error) { + limit, err := pagination.ParseLimit(c.Query("limit")) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, err.Error(), err) + } + token, present := c.GetQuery("cursor") + if !present { + return limit, nil, nil + } + last, err := pagination.DecodeCursor(token, scope) + if err != nil { + return 0, nil, apperr.Wrap(apperr.KindInvalid, "cursor is invalid for this resource", err) + } + return limit, &last, nil +} + +// RawMessage distinguishes an omitted optional array from an explicit null. +func decodeAttachmentIDs(raw json.RawMessage) ([]uuid.UUID, error) { + if raw == nil { + return nil, nil + } + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, apperr.Invalid("attachment_ids must be an array") + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil { + return nil, apperr.Invalid("attachment_ids must be an array of UUIDs") + } + if len(values) > MaxAttachments { + return nil, apperr.Invalid("attachment_ids must have at most 10 entries") + } + ids := make([]uuid.UUID, 0, len(values)) + for _, value := range values { + id, err := ident.Parse(value) + if err != nil { + return nil, apperr.Invalid("attachment_ids must contain valid UUIDs") + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/internal/chat/message.go b/internal/chat/message.go new file mode 100644 index 00000000..cda9b940 --- /dev/null +++ b/internal/chat/message.go @@ -0,0 +1,69 @@ +package chat + +import ( + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxTextCodePoints = 4000 + MaxAttachments = 10 +) + +type Message struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + AttachmentIDs []uuid.UUID +} + +// NewMessage keeps text and attachment order verbatim; it owns its attachment slice. +func NewMessage(id, chatID, senderID uuid.UUID, text string, attachmentIDs ...uuid.UUID) (Message, error) { + if len(attachmentIDs) > MaxAttachments { + return Message{}, apperr.Invalid("attachment_ids must have at most 10 entries") + } + seen := make(map[uuid.UUID]struct{}, len(attachmentIDs)) + for _, id := range attachmentIDs { + if id == uuid.Nil { + return Message{}, apperr.Invalid("attachment_ids must contain non-nil UUIDs") + } + if _, duplicate := seen[id]; duplicate { + return Message{}, apperr.Invalid("attachment_ids must be unique") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 { + return Message{}, apperr.Invalid("text must not be blank") + } + if utf8.RuneCountInString(text) > MaxTextCodePoints { + return Message{}, apperr.Invalid(fmt.Sprintf("text must be at most %d characters", MaxTextCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(text, 0) { + return Message{}, apperr.Invalid("text must not contain the NUL character") + } + return Message{ + ID: id, + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: ident.TimeOf(id), + AttachmentIDs: slices.Clone(attachmentIDs), + }, nil +} + +// SameContent reports whether two messages say the same thing, whatever their +// ids and times. +func (m Message) SameContent(other Message) bool { + return m.Text == other.Text && slices.Equal(m.AttachmentIDs, other.AttachmentIDs) +} diff --git a/internal/chat/message_test.go b/internal/chat/message_test.go new file mode 100644 index 00000000..42762154 --- /dev/null +++ b/internal/chat/message_test.go @@ -0,0 +1,60 @@ +package chat_test + +import ( + "strings" + "testing" + + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func TestNewMessageDerivesCreatedAt(t *testing.T) { + id, chatID, alice := ident.New(), ident.New(), ident.New() + m, err := chat.NewMessage(id, chatID, alice, "Привет, мир 👋") + if err != nil { + t.Fatalf("NewMessage error: %v", err) + } + if m.ID != id || m.ChatID != chatID || m.SenderID != alice || m.Text != "Привет, мир 👋" { + t.Fatalf("unexpected message %+v", m) + } + if !m.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", m.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewMessageValidatesTextByCodePoints(t *testing.T) { + chatID, alice := ident.New(), ident.New() + if _, err := chat.NewMessage(ident.New(), chatID, alice, strings.Repeat("я", 4000)); err != nil { + t.Errorf("4000 code points must be accepted: %v", err) + } + for _, text := range []string{"", " ", "\n\t ", strings.Repeat("я", 4001), "nul\x00inside"} { + _, err := chat.NewMessage(ident.New(), chatID, alice, text) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("text %q must be rejected as invalid, got %v", text, err) + } + } +} + +func TestNewMessageKeepsSurroundingWhitespace(t *testing.T) { + m, err := chat.NewMessage(ident.New(), ident.New(), ident.New(), " hi ") + if err != nil { + t.Fatal(err) + } + if m.Text != " hi " { + t.Fatalf("text must be stored verbatim, got %q", m.Text) + } +} + +func TestSameContentComparesWhatTheSenderWrote(t *testing.T) { + chatID, alice := ident.New(), ident.New() + first, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + retry, _ := chat.NewMessage(ident.New(), chatID, alice, "привет") + other, _ := chat.NewMessage(ident.New(), chatID, alice, "привет!") + if !first.SameContent(retry) { + t.Fatal("messages with the same text must have the same content") + } + if first.SameContent(other) { + t.Fatal("messages with different text must differ") + } +} diff --git a/internal/chat/repository.go b/internal/chat/repository.go new file mode 100644 index 00000000..47d929d7 --- /dev/null +++ b/internal/chat/repository.go @@ -0,0 +1,36 @@ +package chat + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +// ChatRepository stores chats and their membership. Membership is answered +// by queries; the projection with members is assembled by the store in one +// query, never by loading a member list into the entity. Members are always +// returned ordered by id. +type ChatRepository interface { + // Create stores the chat row and its members atomically. + Create(ctx context.Context, chat Chat, memberIDs []uuid.UUID) error + // GetForMember returns the chat with members, or ErrNotFound when the + // chat does not exist or the user is not a member. + GetForMember(ctx context.Context, chatID, userID uuid.UUID) (ChatWithMembers, error) + IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) + // ListByMember returns up to limit chats of the user ordered by + // (created_at, id), strictly after the chat identified by after when set. + ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]ChatWithMembers, error) +} + +type MessageRepository interface { + // AppendOnce stores message under the sender's key unless the key was + // used in this chat within ttl, and returns the message the key is bound + // to: the given one or the earlier one. + AppendOnce(ctx context.Context, message Message, key string, ttl time.Duration) (Message, error) + DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) + // ListByChat returns up to limit messages ordered by (created_at, id), + // strictly after the message identified by after when set. An unknown + // chat yields an empty history. + ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]Message, error) +} diff --git a/internal/chat/service.go b/internal/chat/service.go new file mode 100644 index 00000000..39a66f7c --- /dev/null +++ b/internal/chat/service.go @@ -0,0 +1,140 @@ +package chat + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +const ( + keyPurgeBatch = 1000 + // keyPurgeGrace keeps a key past its TTL, so a key that a request has just + // found valid cannot vanish before the request reads its message back. + keyPurgeGrace = time.Minute +) + +// UserLookup is what this module needs from the users module: one batched +// existence check. It is declared here, on the consumer side, so this +// package never imports the users package and tests substitute a map. +type UserLookup interface { + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} + +type Service struct { + chats ChatRepository + messages MessageRepository + users UserLookup + keyTTL time.Duration +} + +// NewService takes keyTTL, how long an Idempotency-Key stays bound to its +// message. +func NewService(chats ChatRepository, messages MessageRepository, users UserLookup, keyTTL time.Duration) *Service { + return &Service{chats: chats, messages: messages, users: users, keyTTL: keyTTL} +} + +func (s *Service) Create(ctx context.Context, creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatWithMembers, error) { + chat, err := NewChat(ident.New(), title) + if err != nil { + return ChatWithMembers{}, err + } + members, err := ValidateMembers(memberIDs, creator) + if err != nil { + return ChatWithMembers{}, err + } + found, err := s.users.ExistingIDs(ctx, members) + if err != nil { + return ChatWithMembers{}, fmt.Errorf("check members: %w", err) + } + if len(found) != len(members) { + return ChatWithMembers{}, apperr.Invalid("member_ids must reference registered users") + } + if err := s.chats.Create(ctx, chat, members); err != nil { + return ChatWithMembers{}, err + } + return ChatWithMembers{Chat: chat, MemberIDs: members}, nil +} + +func (s *Service) Get(ctx context.Context, actor, chatID uuid.UUID) (ChatWithMembers, error) { + return s.chats.GetForMember(ctx, chatID, actor) +} + +func (s *Service) List(ctx context.Context, actor uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[ChatWithMembers], error) { + items, err := s.chats.ListByMember(ctx, actor, after, limit+1) + if err != nil { + return pagination.Page[ChatWithMembers]{}, err + } + return pagination.Cut(items, limit, func(c ChatWithMembers) uuid.UUID { return c.ID }), nil +} + +// MessagePayload is one send attempt: the message and its Idempotency-Key. +type MessagePayload struct { + ChatID uuid.UUID + IdempotencyKey string + Text string + AttachmentIDs []uuid.UUID +} + +// Send checks access and content on every attempt, a replay included, as the +// contract requires of a repeat. +func (s *Service) Send(ctx context.Context, actor uuid.UUID, p MessagePayload) (Message, error) { + if err := s.requireMember(ctx, actor, p.ChatID); err != nil { + return Message{}, err + } + message, err := NewMessage(ident.New(), p.ChatID, actor, p.Text, p.AttachmentIDs...) + if err != nil { + return Message{}, err + } + stored, err := s.messages.AppendOnce(ctx, message, p.IdempotencyKey, s.keyTTL) + if err != nil { + return Message{}, err + } + if !stored.SameContent(message) { + return Message{}, ErrKeyReused + } + return stored, nil +} + +func (s *Service) PurgeExpiredKeys(ctx context.Context) (int64, error) { + var total int64 + for { + n, err := s.messages.DeleteExpiredKeys(ctx, s.keyTTL+keyPurgeGrace, keyPurgeBatch) + total += n + if err != nil { + return total, err + } + if n < keyPurgeBatch { + return total, nil + } + } +} + +func (s *Service) History(ctx context.Context, actor, chatID uuid.UUID, after *uuid.UUID, limit int) (pagination.Page[Message], error) { + if err := s.requireMember(ctx, actor, chatID); err != nil { + return pagination.Page[Message]{}, err + } + items, err := s.messages.ListByChat(ctx, chatID, after, limit+1) + if err != nil { + return pagination.Page[Message]{}, err + } + return pagination.Cut(items, limit, func(m Message) uuid.UUID { return m.ID }), nil +} + +// requireMember runs before any field validation, so an outsider gets the +// same answer for a foreign chat as for a missing one. +func (s *Service) requireMember(ctx context.Context, actor, chatID uuid.UUID) error { + ok, err := s.chats.IsMember(ctx, chatID, actor) + if err != nil { + return fmt.Errorf("check membership: %w", err) + } + if !ok { + return ErrNotFound + } + return nil +} diff --git a/internal/chat/service_test.go b/internal/chat/service_test.go new file mode 100644 index 00000000..c7a16681 --- /dev/null +++ b/internal/chat/service_test.go @@ -0,0 +1,392 @@ +package chat_test + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +// countingUsers passes through to the real users service and counts the +// lookups, which proves that member checks are batched. +type countingUsers struct { + *user.Service + calls atomic.Int32 +} + +func (c *countingUsers) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + c.calls.Add(1) + return c.Service.ExistingIDs(ctx, ids) +} + +type fixture struct { + svc *chat.Service + users *countingUsers + alice, bob uuid.UUID + eve uuid.UUID + ctx context.Context +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + ctx := context.Background() + users := &countingUsers{Service: user.NewService(userstore.NewUsers(db.Pool))} + register := func(name string) uuid.UUID { + u, err := users.Register(ctx, name, name, "correct-horse-battery") + if err != nil { + t.Fatal(err) + } + return u.ID + } + return fixture{ + svc: chat.NewService(store.NewChats(db.Pool), store.NewMessages(db.Pool), users, time.Hour), + users: users, + alice: register("alice"), + bob: register("bob"), + eve: register("eve"), + ctx: ctx, + } +} + +func TestCreateChatChecksMembersInOneCall(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "Проект", []uuid.UUID{f.eve, f.alice, f.bob}) + if err != nil { + t.Fatalf("Create error: %v", err) + } + if c.Title != "Проект" || len(c.MemberIDs) != 3 { + t.Fatalf("unexpected chat %+v", c) + } + if got := f.users.calls.Load(); got != 1 { + t.Fatalf("member existence must be checked with one batch call, got %d", got) + } + got, err := f.svc.Get(f.ctx, f.bob, c.ID) + if err != nil { + t.Fatalf("member Get error: %v", err) + } + if fmt.Sprint(got.MemberIDs) != fmt.Sprint(c.MemberIDs) || !got.CreatedAt.Equal(c.CreatedAt) || got.Title != c.Title { + t.Fatalf("Get = %+v, want what Create returned %+v", got, c) + } +} + +func TestCreateChatRejectsUnknownMember(t *testing.T) { + f := newFixture(t) + _, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, ident.New()}) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("kind = %v, want KindInvalid (%v)", apperr.KindOf(err), err) + } + if !strings.Contains(err.Error(), "member_ids") { + t.Fatalf("message must name the field: %v", err) + } +} + +func TestCreateChatRejectsInvalidInputBeforeAnyLookup(t *testing.T) { + f := newFixture(t) + cases := map[string]struct { + title string + members []uuid.UUID + }{ + "blank title": {" ", []uuid.UUID{f.alice}}, + "missing creator": {"t", []uuid.UUID{f.bob}}, + "duplicate": {"t", []uuid.UUID{f.alice, f.alice}}, + } + for name, tc := range cases { + _, err := f.svc.Create(f.ctx, f.alice, tc.title, tc.members) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Errorf("%s: kind = %v, want KindInvalid (%v)", name, apperr.KindOf(err), err) + } + } + if got := f.users.calls.Load(); got != 0 { + t.Fatalf("invalid input must be rejected before any user lookup, got %d calls", got) + } +} + +func TestGetHidesForeignAndUnknownChatsAlike(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, foreign := f.svc.Get(f.ctx, f.eve, c.ID) + _, unknown := f.svc.Get(f.ctx, f.eve, ident.New()) + if !errors.Is(foreign, chat.ErrNotFound) || !errors.Is(unknown, chat.ErrNotFound) { + t.Fatalf("foreign = %v, unknown = %v; want both ErrNotFound", foreign, unknown) + } + if apperr.KindOf(foreign) != apperr.KindNotFound { + t.Fatalf("ErrNotFound must carry KindNotFound, got %v", apperr.KindOf(foreign)) + } +} + +func TestListPagesOnlyOwnChats(t *testing.T) { + f := newFixture(t) + var mine []uuid.UUID + for range 3 { + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + mine = append(mine, c.ID) + if _, err := f.svc.Create(f.ctx, f.eve, "t", []uuid.UUID{f.eve}); err != nil { + t.Fatal(err) + } + } + page, err := f.svc.List(f.ctx, f.alice, nil, 2) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 2 || page.Next == nil || *page.Next != mine[1] { + t.Fatalf("first page = %+v", page) + } + if len(page.Items[0].MemberIDs) != 2 { + t.Fatalf("listed chats must carry their members: %+v", page.Items[0]) + } + rest, err := f.svc.List(f.ctx, f.alice, page.Next, 2) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || rest.Items[0].ID != mine[2] || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + all, err := f.svc.List(f.ctx, f.bob, nil, 100) + if err != nil || len(all.Items) != 3 || all.Next != nil { + t.Fatalf("bob page = %+v, %v", all, err) + } +} + +func TestSendChecksAccessBeforeValidation(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + _, err = f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: " "}) + if !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with blank text: err = %v, want ErrNotFound", err) + } + _, err = f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: " "}) + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("member with blank text: kind = %v, want KindInvalid", apperr.KindOf(err)) + } + msg, err := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k3", Text: "Привет 👋"}) + if err != nil || msg.SenderID != f.bob || msg.ChatID != c.ID || msg.Text != "Привет 👋" { + t.Fatalf("Send = %+v, %v", msg, err) + } +} + +func TestHistoryPagesAndHidesForeignChat(t *testing.T) { + f := newFixture(t) + c, err := f.svc.Create(f.ctx, f.alice, "t", []uuid.UUID{f.alice, f.bob}) + if err != nil { + t.Fatal(err) + } + first, _ := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k1", Text: "1"}) + second, _ := f.svc.Send(f.ctx, f.bob, chat.MessagePayload{ChatID: c.ID, IdempotencyKey: "k2", Text: "2"}) + + page, err := f.svc.History(f.ctx, f.bob, c.ID, nil, 1) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || !reflect.DeepEqual(page.Items[0], first) || page.Next == nil || *page.Next != first.ID { + t.Fatalf("first page = %+v", page) + } + rest, err := f.svc.History(f.ctx, f.alice, c.ID, page.Next, 1) + if err != nil { + t.Fatal(err) + } + if len(rest.Items) != 1 || !reflect.DeepEqual(rest.Items[0], second) || rest.Next != nil { + t.Fatalf("second page = %+v", rest) + } + if _, err := f.svc.History(f.ctx, f.eve, c.ID, nil, 10); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider history: err = %v, want ErrNotFound", err) + } + empty, err := f.svc.History(f.ctx, f.alice, c.ID, &second.ID, 10) + if err != nil || empty.Items == nil || len(empty.Items) != 0 || empty.Next != nil { + t.Fatalf("page after last = %+v, %v; want empty non-nil items", empty, err) + } +} + +func (f fixture) newChat(t *testing.T, members ...uuid.UUID) uuid.UUID { + t.Helper() + c, err := f.svc.Create(f.ctx, members[0], "t", members) + if err != nil { + t.Fatal(err) + } + return c.ID +} + +func (f fixture) send(t *testing.T, actor, c uuid.UUID, key, text string) chat.Message { + t.Helper() + m, err := f.svc.Send(f.ctx, actor, chat.MessagePayload{ChatID: c, IdempotencyKey: key, Text: text}) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestSendReplaysSameBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "привет") + again := f.send(t, f.alice, c, "k", "привет") + if again.ID != first.ID || !again.CreatedAt.Equal(first.CreatedAt) { + t.Fatalf("replay = %+v, want %+v", again, first) + } +} + +func TestSendConflictsOnDifferentBody(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "первое") + _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "другое"}) + if !errors.Is(err, chat.ErrKeyReused) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("different body under a used key: err = %v, want ErrKeyReused (conflict)", err) + } +} + +func TestKeysAreScopedByUserAndChat(t *testing.T) { + f := newFixture(t) + c1 := f.newChat(t, f.alice, f.bob) + c2 := f.newChat(t, f.alice, f.bob) + a1 := f.send(t, f.alice, c1, "k", "текст") + b1 := f.send(t, f.bob, c1, "k", "текст") + a2 := f.send(t, f.alice, c2, "k", "текст") + if a1.ID == b1.ID || a1.ID == a2.ID || b1.ID == a2.ID { + t.Fatalf("one key shared across users or chats: %v %v %v", a1.ID, b1.ID, a2.ID) + } +} + +// A replay checks access again: an outsider presenting a member's key learns +// nothing and the key stays bound to the member's message. +func TestSendChecksAccessBeforeKey(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + first := f.send(t, f.alice, c, "k", "секрет") + if _, err := f.svc.Send(f.ctx, f.eve, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: "секрет"}); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider with a used key: err = %v, want ErrNotFound", err) + } + if again := f.send(t, f.alice, c, "k", "секрет"); again.ID != first.ID { + t.Fatalf("key rebound after the outsider: %v, want %v", again.ID, first.ID) + } +} + +func TestInvalidTextWithUsedKeyIsInvalid(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + f.send(t, f.alice, c, "k", "текст") + if _, err := f.svc.Send(f.ctx, f.alice, chat.MessagePayload{ChatID: c, IdempotencyKey: "k", Text: " "}); apperr.KindOf(err) != apperr.KindInvalid { + t.Fatalf("blank text under a used key: kind = %v, want KindInvalid", apperr.KindOf(err)) + } +} + +func TestPurgeExpiredKeysDeletesInBatches(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice, f.bob) + const expired = 2500 + _, err := db.Pool.Exec(f.ctx, ` + WITH m AS ( + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $1, gen_random_uuid(), $2, 'old', now() FROM generate_series(1, $3) + RETURNING id) + INSERT INTO message_idempotency_keys (user_id, chat_id, key, message_id, created_at) + SELECT $2, $1, id::text, id, now() - interval '2 hours' FROM m`, c, f.alice, expired) + if err != nil { + t.Fatal(err) + } + f.send(t, f.alice, c, "fresh", "новое") + deleted, err := f.svc.PurgeExpiredKeys(f.ctx) + if err != nil || deleted != expired { + t.Fatalf("PurgeExpiredKeys = %d, %v; want %d", deleted, err, expired) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want the fresh one", left, err) + } +} + +// A key the claim has just seen valid must survive until the replay lookup, +// so the cleanup leaves keys alone for a while after they expire. +func TestPurgeExpiredKeysWaitsAGraceAfterTTL(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "just-expired", "1") + f.send(t, f.alice, c, "long-expired", "2") + if _, err := db.Pool.Exec(f.ctx, ` + UPDATE message_idempotency_keys + SET created_at = now() - interval '1 hour' - CASE key WHEN 'just-expired' THEN interval '10 seconds' ELSE interval '10 minutes' END`); err != nil { + t.Fatal(err) + } + if deleted, err := f.svc.PurgeExpiredKeys(f.ctx); err != nil || deleted != 1 { + t.Fatalf("PurgeExpiredKeys = %d, %v; want only the long expired key", deleted, err) + } + var left string + if err := db.Pool.QueryRow(f.ctx, `SELECT key FROM message_idempotency_keys`).Scan(&left); err != nil || left != "just-expired" { + t.Fatalf("left %q, %v; want just-expired", left, err) + } +} + +func TestRunKeyCleanupPurgesAndStops(t *testing.T) { + f := newFixture(t) + c := f.newChat(t, f.alice) + f.send(t, f.alice, c, "old", "старое") + if _, err := db.Pool.Exec(f.ctx, `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(f.ctx) + stopped := make(chan struct{}) + go func() { + chat.RunKeyCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), f.svc, 50*time.Millisecond) + close(stopped) + }() + deadline := time.Now().Add(2 * time.Second) + for { + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil { + t.Fatal(err) + } + if left == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("expired key was not purged within 2s") + } + time.Sleep(20 * time.Millisecond) + } + cancel() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("RunKeyCleanup did not return after cancel") + } +} diff --git a/internal/chat/store/attachments_test.go b/internal/chat/store/attachments_test.go new file mode 100644 index 00000000..d9cdc265 --- /dev/null +++ b/internal/chat/store/attachments_test.go @@ -0,0 +1,399 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/pipeline" + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +func (f fixture) attachment(t *testing.T, chatID, owner uuid.UUID, status string) uuid.UUID { + t.Helper() + id := ident.New() + hash := sha256.Sum256([]byte(id.String())) + _, err := db.Pool.Exec(f.ctx, ` + WITH file AS ( + INSERT INTO files (hash, size_bytes, mime_type, status, created_at) + VALUES ($1, 7, 'image/png', $2, now()) RETURNING hash) + INSERT INTO attachments (id, chat_id, owner_id, filename, file_hash, created_at) + SELECT $3, $4, $5, 'image.png', hash, now() FROM file`, + hash[:], "pending", id, chatID, owner) + if err != nil { + t.Fatal(err) + } + _, err = db.Pool.Exec(f.ctx, `INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code) + VALUES ($1,$2,$3,CASE WHEN $3='failed' THEN 'source_missing' END)`, id, "staging/"+id.String(), status) + if err != nil { + t.Fatal(err) + } + return id +} + +func attachedMessage(t *testing.T, c, sender uuid.UUID, ids ...uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), c, sender, "", ids...) + if err != nil { + t.Fatal(err) + } + return m +} + +func assertCounts(t *testing.T, messages, keys, joins int) { + t.Helper() + var gotMessages, gotKeys, gotJoins int + err := db.Pool.QueryRow(context.Background(), ` + SELECT (SELECT count(*) FROM messages), + (SELECT count(*) FROM message_idempotency_keys), + (SELECT count(*) FROM message_attachments)`).Scan(&gotMessages, &gotKeys, &gotJoins) + if err != nil || gotMessages != messages || gotKeys != keys || gotJoins != joins { + t.Fatalf("counts = %d/%d/%d, %v; want %d/%d/%d", gotMessages, gotKeys, gotJoins, err, messages, keys, joins) + } +} + +func TestAppendOnceRejectsUnknownAttachmentWithoutClaim(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + m := attachedMessage(t, c.ID, alice, ident.New()) + _, err := f.messages.AppendOnce(f.ctx, m, "unknown", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("unknown attachment: %v; want not found", err) + } + var messages, keys int + err = db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM messages), (SELECT count(*) FROM message_idempotency_keys)`).Scan(&messages, &keys) + if err != nil || messages != 0 || keys != 0 { + t.Fatalf("failed validation persisted %d messages/%d keys: %v", messages, keys, err) + } +} + +func TestAppendOnceAcceptsQueuedAttachmentDuringPublish(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + id := f.attachment(t, c.ID, alice, "queued") + m := attachedMessage(t, c.ID, alice, id) + found, err := pipeline.NewStore(db.Pool).PublishOne(f.ctx, func(ctx context.Context, j pipeline.Job) error { + if j.AttachmentID != id { + t.Fatalf("publisher selected %s, want %s", j.AttachmentID, id) + } + // The callback holds the publisher transaction open until AppendOnce returns. + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + _, err := f.messages.AppendOnce(ctx, m, "during-publish", keyTTL) + return err + }) + if err != nil || !found { + t.Fatalf("linking queued attachment while awaiting broker confirm: found=%v, err=%v", found, err) + } + assertCounts(t, 1, 1, 1) +} + +func TestAppendOnceSeesFailureCommittedWhileValidating(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + id := f.attachment(t, c.ID, alice, "queued") + if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state='processing', + lease_token=gen_random_uuid(),lease_until=now()+interval '1 hour' WHERE attachment_id=$1`, id); err != nil { + t.Fatal(err) + } + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(f.ctx) + // Fail and lease recovery both take this lock before deciding the terminal state. + if _, err := tx.Exec(f.ctx, `SELECT job_id FROM attachment_jobs WHERE attachment_id=$1 FOR UPDATE`, id); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(f.ctx, `UPDATE attachment_jobs SET state='failed',error_code='retry_exhausted', + lease_token=NULL,lease_until=NULL WHERE attachment_id=$1`, id); err != nil { + t.Fatal(err) + } + m := attachedMessage(t, c.ID, alice, id) + ctx, cancel := context.WithTimeout(f.ctx, 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := f.messages.AppendOnce(ctx, m, "failing", keyTTL) + done <- err + }() + for { + var waiting bool + err := db.Pool.QueryRow(ctx, `SELECT EXISTS ( + SELECT FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid)))`, tx.Conn().PgConn().PID()).Scan(&waiting) + if err != nil { + t.Fatal(err) + } + if waiting { + break + } + select { + case err := <-done: + t.Fatalf("message finished before the terminal transition: %v", err) + case <-time.After(10 * time.Millisecond): + } + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; err == nil || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("message with concurrently failed attachment: %v; want conflict", err) + } + assertCounts(t, 0, 0, 0) +} + +func TestAppendOnceAttachmentValidationIsAtomic(t *testing.T) { + for _, status := range []string{"foreign", "failed", "mixed"} { + t.Run(status, func(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + other := f.mustChat(t, alice) + ready := f.attachment(t, c.ID, bob, "queued") + var invalid uuid.UUID + want := apperr.KindConflict + switch status { + case "foreign": + invalid = f.attachment(t, other.ID, alice, "queued") + want = apperr.KindNotFound + case "mixed": + ready = f.attachment(t, c.ID, bob, "failed") + invalid = ident.New() + want = apperr.KindNotFound + default: + invalid = f.attachment(t, c.ID, alice, status) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ready, invalid), "bad", keyTTL) + if err == nil || apperr.KindOf(err) != want { + t.Fatalf("validation: %v; want %v", err, want) + } + assertCounts(t, 0, 0, 0) + good := f.attachment(t, c.ID, bob, "queued") + f.appendOnce(t, attachedMessage(t, c.ID, alice, good), "bad") + assertCounts(t, 1, 1, 1) + }) + } +} + +func TestAppendOnceAttachmentOrderReplayAndExpiration(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + a, b := f.attachment(t, c.ID, bob, "queued"), f.attachment(t, c.ID, bob, "queued") + first := attachedMessage(t, c.ID, alice, b, a) + got := f.appendOnce(t, first, "k") + if !reflect.DeepEqual(got, first) { + t.Fatalf("append = %+v, want %+v", got, first) + } + replayed := f.appendOnce(t, attachedMessage(t, c.ID, alice, a, b), "k") + if !reflect.DeepEqual(replayed, first) { + t.Fatalf("replay must return original ordered attachments: %+v", replayed) + } + assertCounts(t, 1, 1, 2) + expire(t) + second := attachedMessage(t, c.ID, bob, a) + f.appendOnce(t, second, "k2") + replacement := attachedMessage(t, c.ID, alice, a, b) + f.appendOnce(t, replacement, "k") + assertCounts(t, 3, 2, 5) + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 1) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("first page = %+v, %v", page, err) + } + page, err = f.messages.ListByChat(f.ctx, c.ID, &first.ID, 10) + if err != nil || !reflect.DeepEqual(page, []chat.Message{second, replacement}) { + t.Fatalf("remaining page = %+v, %v", page, err) + } +} + +func TestAppendOnceReplaysAcceptedMessageAfterAttachmentFails(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "queued") + first := attachedMessage(t, c.ID, alice, a) + f.appendOnce(t, first, "k") + if _, err := db.Pool.Exec(f.ctx, `UPDATE attachment_jobs SET state = 'failed',error_code='source_missing'`); err != nil { + t.Fatal(err) + } + got := f.appendOnce(t, attachedMessage(t, c.ID, alice, a), "k") + if !reflect.DeepEqual(got, first) { + t.Fatalf("replay = %+v, want original %+v", got, first) + } + _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "new", keyTTL) + if err == nil || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("new message with failed attachment = %v, want conflict", err) + } + page, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 1 || !reflect.DeepEqual(page[0], first) { + t.Fatalf("failed blob must preserve history attachment ids: %+v, %v", page, err) + } + assertCounts(t, 1, 1, 1) +} + +func TestAppendOnceRollsBackClaimAndMessageOnJoinFailure(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.attachment(t, c.ID, alice, "queued") + // A database fault at the join boundary must not leave a key or message. + _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments ADD CONSTRAINT test_reject_join CHECK (ordinal < 0)`) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `ALTER TABLE message_attachments DROP CONSTRAINT test_reject_join`); err != nil { + t.Error(err) + } + }() + if _, err := f.messages.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, a), "k", keyTTL); err == nil { + t.Fatal("join fault must fail append") + } + assertCounts(t, 0, 0, 0) +} + +func TestConcurrentAttachmentAppendsSameKeyStoreOneOrderedSet(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a, b := f.attachment(t, c.ID, alice, "queued"), f.attachment(t, c.ID, alice, "queued") + const requests = 20 + results := make(chan chat.Message, requests) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range requests { + ids := []uuid.UUID{a, b} + if i%2 != 0 { + ids = []uuid.UUID{b, a} + } + m := attachedMessage(t, c.ID, alice, ids...) + wg.Go(func() { + <-start + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + results <- got + }) + } + close(start) + wg.Wait() + close(results) + var winner *chat.Message + n := 0 + for got := range results { + n++ + if winner == nil { + winner = &got + } else if !reflect.DeepEqual(got, *winner) { + t.Fatalf("concurrent replay differs: %+v, want %+v", got, *winner) + } + } + if n != requests { + t.Fatalf("got %d successful replies, want %d", n, requests) + } + assertCounts(t, 1, 1, 2) +} + +type queryCounter struct{ count atomic.Int32 } + +func (c *queryCounter) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + c.count.Add(1) + return ctx +} +func (*queryCounter) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} + +func TestAttachmentHistorySingleQueryAndBatchedValidation(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + var ids []uuid.UUID + for range 10 { + ids = append(ids, f.attachment(t, c.ID, alice, "queued")) + } + cfg := db.Pool.Config() + counter := &queryCounter{} + cfg.ConnConfig.Tracer = counter + pool, err := pgxpool.NewWithConfig(f.ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := pool.Ping(f.ctx); err != nil { + t.Fatal(err) + } + repo := store.NewMessages(pool) + counter.count.Store(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids[0]), "one", keyTTL); err != nil { + t.Fatal(err) + } + one := counter.count.Swap(0) + if _, err := repo.AppendOnce(f.ctx, attachedMessage(t, c.ID, alice, ids...), "ten", keyTTL); err != nil { + t.Fatal(err) + } + ten := counter.count.Swap(0) + if one != ten { + t.Fatalf("queries grew from %d to %d for 1 vs 10 attachments", one, ten) + } + page, err := repo.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil || len(page) != 2 || !reflect.DeepEqual(page[1].AttachmentIDs, ids) { + t.Fatalf("history = %+v, %v", page, err) + } + if n := counter.count.Load(); n != 1 { + t.Fatalf("history used %d queries, want one", n) + } +} + +func TestAppendOnceRetriesWhenBoundKeyDisappears(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + f.appendOnce(t, first, "vanish") + next := f.newMessage(t, c.ID, alice) + // Simulate the missing-key boundary after an unsuccessful claim using a + // statement trigger. Its second invocation sees no key and is a no-op. + _, err := db.Pool.Exec(f.ctx, fmt.Sprintf(` + CREATE FUNCTION test_vanish_key() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM messages WHERE id = '%s'::uuid) THEN + DELETE FROM message_idempotency_keys WHERE key = 'vanish'; + END IF; + RETURN NULL; + END $$; + CREATE TRIGGER test_vanish_key AFTER INSERT ON messages + FOR EACH STATEMENT EXECUTE FUNCTION test_vanish_key()`, next.ID.String())) + if err != nil { + t.Fatal(err) + } + defer func() { + if _, err := db.Pool.Exec(f.ctx, `DROP TRIGGER test_vanish_key ON messages; DROP FUNCTION test_vanish_key()`); err != nil { + t.Error(err) + } + }() + ctx, cancel := context.WithTimeout(f.ctx, 3*time.Second) + defer cancel() + got, err := f.messages.AppendOnce(ctx, next, "vanish", keyTTL) + if err != nil || got.ID != next.ID { + t.Fatalf("missing bound key = %+v, %v; want new message", got, err) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } +} diff --git a/internal/chat/store/idempotency_test.go b/internal/chat/store/idempotency_test.go new file mode 100644 index 00000000..2b489947 --- /dev/null +++ b/internal/chat/store/idempotency_test.go @@ -0,0 +1,228 @@ +package store_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" +) + +const keyTTL = time.Hour + +// expire makes every stored key older than keyTTL. +func expire(t *testing.T) { + t.Helper() + if _, err := db.Pool.Exec(context.Background(), + `UPDATE message_idempotency_keys SET created_at = now() - interval '2 hours'`); err != nil { + t.Fatal(err) + } +} + +func (f fixture) appendOnce(t *testing.T, m chat.Message, key string) chat.Message { + t.Helper() + got, err := f.messages.AppendOnce(f.ctx, m, key, keyTTL) + if err != nil { + t.Fatal(err) + } + return got +} + +func (f fixture) count(t *testing.T, chatID uuid.UUID) int { + t.Helper() + got, err := f.messages.ListByChat(f.ctx, chatID, nil, 1000) + if err != nil { + t.Fatal(err) + } + return len(got) +} + +func keyTarget(t *testing.T, key string) uuid.UUID { + t.Helper() + var id uuid.UUID + if err := db.Pool.QueryRow(context.Background(), + `SELECT message_id FROM message_idempotency_keys WHERE key = $1`, key).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +func TestAppendOnceInsertsThenReplays(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, first, "k1"); got.ID != first.ID { + t.Fatalf("first append returned %v, want the new message %v", got.ID, first.ID) + } + retry := f.newMessage(t, c.ID, alice) + got := f.appendOnce(t, retry, "k1") + if got.ID != first.ID || !got.CreatedAt.Equal(first.CreatedAt) || got.Text != first.Text || got.SenderID != alice { + t.Fatalf("replay returned %+v, want the stored %+v", got, first) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} + +func TestAppendOnceTakesOverExpiredKey(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + next := f.newMessage(t, c.ID, alice) + if got := f.appendOnce(t, next, "k1"); got.ID != next.ID { + t.Fatalf("expired key returned %v, want the new message %v", got.ID, next.ID) + } + if n := f.count(t, c.ID); n != 2 { + t.Fatalf("history has %d messages, want 2", n) + } + if target := keyTarget(t, "k1"); target != next.ID { + t.Fatalf("key points to %v, want %v", target, next.ID) + } +} + +// A cleaner batch holds the expired key row; the request waits for it and, +// once the row is gone, inserts its message under the now free key. +func TestAppendOnceWaitsForCleanerThenInserts(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(f.ctx) + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + + next := f.newMessage(t, c.ID, alice) + type result struct { + m chat.Message + err error + } + done := make(chan result, 1) + go func() { + m, err := f.messages.AppendOnce(f.ctx, next, "k1", keyTTL) + done <- result{m, err} + }() + select { + case r := <-done: + t.Fatalf("AppendOnce did not wait for the locked key: %+v", r) + case <-time.After(300 * time.Millisecond): + } + if _, err := tx.Exec(f.ctx, `DELETE FROM message_idempotency_keys WHERE key = 'k1'`); err != nil { + t.Fatal(err) + } + if err := tx.Commit(f.ctx); err != nil { + t.Fatal(err) + } + select { + case r := <-done: + if r.err != nil || r.m.ID != next.ID { + t.Fatalf("after the cleaner: %+v, %v; want the new message %v", r.m, r.err, next.ID) + } + case <-time.After(5 * time.Second): + t.Fatal("AppendOnce did not finish after the cleaner committed") + } +} + +// A request holding a key row must not make the cleaner wait: it skips the +// row and deletes it on a later run. +func TestCleanerSkipsRowLockedByRequest(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "k1") + expire(t) + + tx, err := db.Pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(f.ctx, `SELECT 1 FROM message_idempotency_keys WHERE key = 'k1' FOR UPDATE`); err != nil { + t.Fatal(err) + } + start := time.Now() + n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000) + if err != nil || n != 0 { + t.Fatalf("cleaner over a locked row: deleted %d, %v; want 0", n, err) + } + if waited := time.Since(start); waited > time.Second { + t.Fatalf("cleaner waited %v for the locked row", waited) + } + if err := tx.Rollback(f.ctx); err != nil { + t.Fatal(err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 1000); err != nil || n != 1 { + t.Fatalf("cleaner after release: deleted %d, %v; want 1", n, err) + } +} + +func TestDeleteExpiredKeysOnlyExpiredAndUpToLimit(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + for _, k := range []string{"a", "b", "c"} { + f.appendOnce(t, f.newMessage(t, c.ID, alice), k) + } + expire(t) + f.appendOnce(t, f.newMessage(t, c.ID, alice), "fresh") + + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 2 { + t.Fatalf("first batch deleted %d, %v; want 2", n, err) + } + if n, err := f.messages.DeleteExpiredKeys(f.ctx, keyTTL, 2); err != nil || n != 1 { + t.Fatalf("second batch deleted %d, %v; want 1", n, err) + } + var left int + if err := db.Pool.QueryRow(f.ctx, `SELECT count(*) FROM message_idempotency_keys`).Scan(&left); err != nil || left != 1 { + t.Fatalf("%d keys left, %v; want only the fresh one", left, err) + } + if n := f.count(t, c.ID); n != 4 { + t.Fatalf("cleanup removed messages: %d left, want 4", n) + } +} + +// Every request brings its own new message under one key; exactly one is +// stored and all of them get it back. This also proves the key row and the +// message it references are written by one statement. +func TestConcurrentAppendOnceSameKeyStoresOneMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const requests = 20 + ids := make(chan uuid.UUID, requests) + var wg sync.WaitGroup + for range requests { + m := f.newMessage(t, c.ID, alice) + wg.Go(func() { + got, err := f.messages.AppendOnce(f.ctx, m, "same", keyTTL) + if err != nil { + t.Error(err) + return + } + ids <- got.ID + }) + } + wg.Wait() + close(ids) + seen := map[uuid.UUID]int{} + for id := range ids { + seen[id]++ + } + if len(seen) != 1 { + t.Fatalf("requests got %d different messages, want 1: %v", len(seen), seen) + } + if n := f.count(t, c.ID); n != 1 { + t.Fatalf("history has %d messages, want 1", n) + } +} diff --git a/internal/chat/store/postgres.go b/internal/chat/store/postgres.go new file mode 100644 index 00000000..e8a15d09 --- /dev/null +++ b/internal/chat/store/postgres.go @@ -0,0 +1,291 @@ +// Package store holds the repository implementations of the chats module. +// +// Keyset pages use `id > COALESCE($after, nil uuid)`: every UUIDv7 is +// greater than the nil UUID, so one query serves the first page and the +// following ones, and the condition stays a range the index can scan. Ids +// are UUIDv7 and created_at derives from them, so ordering by id is the +// contract order (created_at, id). Errors pass through postgres.Classify, so +// an unreachable database surfaces as unavailability. +package store + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/chat" + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" +) + +type Chats struct { + pool *pgxpool.Pool +} + +var _ chat.ChatRepository = (*Chats)(nil) + +func NewChats(pool *pgxpool.Pool) *Chats { + return &Chats{pool: pool} +} + +func (r *Chats) Create(ctx context.Context, c chat.Chat, memberIDs []uuid.UUID) error { + return postgres.Classify(pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, + `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`, + c.ID, c.Title, c.CreatedAt); err != nil { + return fmt.Errorf("insert chat: %w", err) + } + if _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id) SELECT $1, unnest($2::uuid[])`, + c.ID, memberIDs); err != nil { + return fmt.Errorf("insert chat members: %w", err) + } + return nil + })) +} + +func (r *Chats) GetForMember(ctx context.Context, chatID, userID uuid.UUID) (chat.ChatWithMembers, error) { + row := r.pool.QueryRow(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chats c + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID) + c, err := scanChat(row) + if errors.Is(err, pgx.ErrNoRows) { + return chat.ChatWithMembers{}, chat.ErrNotFound + } + if err != nil { + return chat.ChatWithMembers{}, fmt.Errorf("select chat: %w", postgres.Classify(err)) + } + return c, nil +} + +func (r *Chats) IsMember(ctx context.Context, chatID, userID uuid.UUID) (bool, error) { + var ok bool + err := r.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + if err != nil { + return false, fmt.Errorf("check membership: %w", postgres.Classify(err)) + } + return ok, nil +} + +func (r *Chats) ListByMember(ctx context.Context, userID uuid.UUID, after *uuid.UUID, limit int) ([]chat.ChatWithMembers, error) { + rows, err := r.pool.Query(ctx, ` + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) + FROM chat_members own + JOIN chats c ON c.id = own.chat_id + WHERE own.user_id = $1 + AND own.chat_id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY own.chat_id + LIMIT $3`, + userID, after, limit) + if err != nil { + return nil, fmt.Errorf("select chats: %w", postgres.Classify(err)) + } + chats, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.ChatWithMembers, error) { + return scanChat(row) + }) + if err != nil { + return nil, fmt.Errorf("read chats: %w", postgres.Classify(err)) + } + return chats, nil +} + +func scanChat(row pgx.Row) (chat.ChatWithMembers, error) { + var c chat.ChatWithMembers + if err := row.Scan(&c.ID, &c.Title, &c.CreatedAt, &c.MemberIDs); err != nil { + return chat.ChatWithMembers{}, err + } + return c, nil +} + +type Messages struct { + pool *pgxpool.Pool +} + +var _ chat.MessageRepository = (*Messages)(nil) + +func NewMessages(pool *pgxpool.Pool) *Messages { + return &Messages{pool: pool} +} + +// claimSQL binds the key and inserts the message in one statement. An expired +// key is taken over; a valid one leaves claimed and the message insert empty. +const claimSQL = ` + WITH claimed AS ( + INSERT INTO message_idempotency_keys AS k (user_id, chat_id, key, message_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET message_id = EXCLUDED.message_id, created_at = now() + WHERE k.created_at < now() - $5::interval + RETURNING message_id) + INSERT INTO messages (chat_id, id, sender_id, text, created_at) + SELECT $2, message_id, $1, $6, $7 FROM claimed` + +// The transaction keeps attachment readiness, message joins and the key atomic. +// The replay uses a new statement snapshot to see a concurrent winner. +func (r *Messages) AppendOnce(ctx context.Context, m chat.Message, key string, ttl time.Duration) (chat.Message, error) { + var stored chat.Message + err := pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + for { + if err := ctx.Err(); err != nil { + return err + } + tag, err := tx.Exec(ctx, claimSQL, + m.SenderID, m.ChatID, key, m.ID, interval(ttl), m.Text, m.CreatedAt) + if err != nil { + return fmt.Errorf("insert message: %w", err) + } + if tag.RowsAffected() == 1 { + // Only a new message creates attachment links. A replay returns the + // committed message even if its attachment has since failed. + if err := validateAttachments(ctx, tx, m.ChatID, m.AttachmentIDs); err != nil { + return err + } + if len(m.AttachmentIDs) > 0 { + _, err := tx.Exec(ctx, ` + INSERT INTO message_attachments (chat_id, message_id, attachment_id, ordinal) + SELECT $1, $2, id, (ord - 1)::smallint + FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)`, + m.ChatID, m.ID, m.AttachmentIDs) + if err != nil { + return fmt.Errorf("insert message attachments: %w", err) + } + } + stored = m + return nil + } + stored, err = scanMessage(tx.QueryRow(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM message_idempotency_keys k + JOIN messages m ON m.chat_id = k.chat_id AND m.id = k.message_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3`, + m.SenderID, m.ChatID, key)) + if errors.Is(err, pgx.ErrNoRows) { + // A vanished key is available again; retry its atomic claim. + continue + } + if err != nil { + return fmt.Errorf("select message by key: %w", err) + } + return nil + } + }) + if err != nil { + return chat.Message{}, postgres.Classify(err) + } + return stored, nil +} + +func validateAttachments(ctx context.Context, tx pgx.Tx, chatID uuid.UUID, ids []uuid.UUID) error { + if len(ids) == 0 { + return nil + } + // Lock attachment identities in the same order for every message. Pipeline + // version changes lock the attachment before replacing its current job. + rows, err := tx.Query(ctx, ` + SELECT id FROM attachments WHERE chat_id=$1 AND id=ANY($2::uuid[]) + ORDER BY id FOR SHARE`, chatID, ids) + if err != nil { + return fmt.Errorf("lock attachments: %w", err) + } + locked, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) + if err != nil { + return err + } + if len(locked) != len(ids) { + return apperr.NotFound("attachment not found") + } + // A separate statement sees the latest committed transition after a lock wait. + // Terminal failures take FOR UPDATE on the job or file. KEY SHARE orders + // validation with those transitions while allowing an outbox publish in flight. + rows, err = tx.Query(ctx, `SELECT status FROM attachment_states WHERE id=ANY($1::uuid[]) ORDER BY id FOR KEY SHARE`, ids) + if err != nil { + return err + } + states, err := pgx.CollectRows(rows, pgx.RowTo[attachment.FileStatus]) + if err != nil { + return err + } + if len(states) != len(ids) { + return apperr.Conflict("attachment processing state unavailable") + } + for _, state := range states { + switch state { + case attachment.FileQueued, attachment.FileProcessing, attachment.FileReady: + default: + return apperr.Conflict("attachment processing failed") + } + } + return nil +} + +func scanMessage(row pgx.Row) (chat.Message, error) { + var m chat.Message + err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs) + if len(m.AttachmentIDs) == 0 { + m.AttachmentIDs = nil + } + return m, err +} + +// DeleteExpiredKeys skips rows locked by another cleaner or a request rather +// than waiting for them. +func (r *Messages) DeleteExpiredKeys(ctx context.Context, olderThan time.Duration, limit int) (int64, error) { + tag, err := r.pool.Exec(ctx, ` + DELETE FROM message_idempotency_keys + WHERE (user_id, chat_id, key) IN ( + SELECT user_id, chat_id, key FROM message_idempotency_keys + WHERE created_at < now() - $1::interval + ORDER BY created_at + LIMIT $2 + FOR UPDATE SKIP LOCKED)`, + interval(olderThan), limit) + if err != nil { + return 0, fmt.Errorf("delete expired keys: %w", postgres.Classify(err)) + } + return tag.RowsAffected(), nil +} + +// interval renders d for a ::interval parameter. +func interval(d time.Duration) string { + return fmt.Sprintf("%d milliseconds", d.Milliseconds()) +} + +// ListByChat keeps keyset pagination on messages and collects each ordered +// attachment array in the same statement. +func (r *Messages) ListByChat(ctx context.Context, chatID uuid.UUID, after *uuid.UUID, limit int) ([]chat.Message, error) { + rows, err := r.pool.Query(ctx, ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.chat_id = m.chat_id AND a.message_id = m.id ORDER BY a.ordinal) + FROM messages m + WHERE m.chat_id = $1 + AND m.id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY id + LIMIT $3`, + chatID, after, limit) + if err != nil { + return nil, fmt.Errorf("select messages: %w", postgres.Classify(err)) + } + messages, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (chat.Message, error) { + return scanMessage(row) + }) + if err != nil { + return nil, fmt.Errorf("read messages: %w", postgres.Classify(err)) + } + return messages, nil +} diff --git a/internal/chat/store/postgres_test.go b/internal/chat/store/postgres_test.go new file mode 100644 index 00000000..d2b01270 --- /dev/null +++ b/internal/chat/store/postgres_test.go @@ -0,0 +1,321 @@ +package store_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "reflect" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/chat/store" + "messenger/internal/platform/ident" + "messenger/internal/testenv" + "messenger/internal/user" + userstore "messenger/internal/user/store" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +type fixture struct { + ctx context.Context + chats *store.Chats + messages *store.Messages +} + +func newFixture(t *testing.T) fixture { + t.Helper() + db.Reset(t) + return fixture{ctx: context.Background(), chats: store.NewChats(db.Pool), messages: store.NewMessages(db.Pool)} +} + +// mustUser stores a user row: chats and messages reference users by FK. +func (f fixture) mustUser(t *testing.T) uuid.UUID { + t.Helper() + id := ident.New() + u, err := user.New(id, "u"+id.String()[24:], "User") + if err != nil { + t.Fatal(err) + } + u.PasswordHash = "x" + if err := userstore.NewUsers(db.Pool).Create(f.ctx, u); err != nil { + t.Fatal(err) + } + return id +} + +func (f fixture) mustChat(t *testing.T, members ...uuid.UUID) chat.Chat { + t.Helper() + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, members); err != nil { + t.Fatal(err) + } + return c +} + +func (f fixture) newMessage(t *testing.T, chatID, sender uuid.UUID) chat.Message { + t.Helper() + m, err := chat.NewMessage(ident.New(), chatID, sender, "text") + if err != nil { + t.Fatal(err) + } + return m +} + +// appendErr stores m under a key of its own, as every send now carries one. +func (f fixture) appendErr(m chat.Message) error { + _, err := f.messages.AppendOnce(f.ctx, m, m.ID.String(), time.Hour) + return err +} + +func TestChatCreateThenGetForMember(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + c := f.mustChat(t, alice, bob) + + got, err := f.chats.GetForMember(f.ctx, c.ID, bob) + if err != nil { + t.Fatal(err) + } + if got.Chat != c || len(got.MemberIDs) != 2 || got.MemberIDs[0] != alice || got.MemberIDs[1] != bob { + t.Fatalf("GetForMember = %+v, want %+v with members [alice bob]", got, c) + } + if _, err := f.chats.GetForMember(f.ctx, c.ID, f.mustUser(t)); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("outsider must get ErrNotFound, got %v", err) + } + for who, want := range map[uuid.UUID]bool{alice: true, bob: true, ident.New(): false} { + ok, err := f.chats.IsMember(f.ctx, c.ID, who) + if err != nil || ok != want { + t.Fatalf("IsMember(%s) = %v, %v; want %v", who, ok, err, want) + } + } +} + +// The chat row and its members are written in one transaction: a failing +// member insert must leave no chat behind. +func TestCreateRollsBackWhenMemberInsertFails(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c, err := chat.NewChat(ident.New(), "half-made") + if err != nil { + t.Fatal(err) + } + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, ident.New()}); err == nil { + t.Fatal("Create with an unknown member must fail") + } + var chats, members int + if err := db.Pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members)`).Scan(&chats, &members); err != nil { + t.Fatal(err) + } + if chats != 0 || members != 0 { + t.Fatalf("chats = %d, members = %d after a failed create; want 0 and 0", chats, members) + } +} + +func TestChatUnknownID(t *testing.T) { + f := newFixture(t) + if _, err := f.chats.GetForMember(f.ctx, ident.New(), ident.New()); !errors.Is(err, chat.ErrNotFound) { + t.Fatalf("GetForMember err = %v, want ErrNotFound", err) + } + if ok, err := f.chats.IsMember(f.ctx, ident.New(), ident.New()); err != nil || ok { + t.Fatalf("IsMember on unknown chat = %v, %v; want false, nil", ok, err) + } +} + +func TestListByMemberFiltersOrdersAndPages(t *testing.T) { + f := newFixture(t) + alice, bob := f.mustUser(t), f.mustUser(t) + var mine []chat.Chat + for range 5 { + c, err := chat.NewChat(ident.New(), "title") + if err != nil { + t.Fatal(err) + } + mine = append(mine, c) + // A chat without alice must never show up in her listing. + f.mustChat(t, bob) + } + // Insert the oldest chat last to prove the store sorts, not appends. + for _, c := range append(mine[1:], mine[0]) { + if err := f.chats.Create(f.ctx, c, []uuid.UUID{alice, bob}); err != nil { + t.Fatal(err) + } + } + + page, err := f.chats.ListByMember(f.ctx, alice, nil, 3) + if err != nil { + t.Fatal(err) + } + if len(page) != 3 || page[0].ID != mine[0].ID || page[2].ID != mine[2].ID { + t.Fatalf("first page mismatch: %v", page) + } + if len(page[0].MemberIDs) != 2 || page[0].MemberIDs[0] != alice { + t.Fatalf("listing must carry members in id order: %+v", page[0]) + } + last := page[2].ID + rest, err := f.chats.ListByMember(f.ctx, alice, &last, 10) + if err != nil { + t.Fatal(err) + } + if len(rest) != 2 || rest[0].ID != mine[3].ID || rest[1].ID != mine[4].ID { + t.Fatalf("second page mismatch: %v", rest) + } + none, err := f.chats.ListByMember(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if none == nil || len(none) != 0 { + t.Fatalf("unknown member must get an empty non-nil slice, got %#v", none) + } +} + +func TestListByChatUnknownChatIsEmpty(t *testing.T) { + f := newFixture(t) + got, err := f.messages.ListByChat(f.ctx, ident.New(), nil, 10) + if err != nil { + t.Fatal(err) + } + if got == nil || len(got) != 0 { + t.Fatalf("want empty non-nil slice, got %#v", got) + } +} + +func TestAppendOutOfOrderIsListedByID(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + first, second, third := f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice), f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{third, first, second} { + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || !reflect.DeepEqual(got[0], first) || !reflect.DeepEqual(got[1], second) || !reflect.DeepEqual(got[2], third) { + t.Fatalf("history must be ordered by id and round-trip every field, got %+v", got) + } +} + +func TestPagingWalksWholeHistoryWithoutGapsOrDuplicates(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const total = 250 + var expected []uuid.UUID + for range total { + m := f.newMessage(t, c.ID, alice) + expected = append(expected, m.ID) + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + // Another chat's history must not leak into this one. + other := f.mustChat(t, alice) + if err := f.appendErr(f.newMessage(t, other.ID, alice)); err != nil { + t.Fatal(err) + } + for _, limit := range []int{1, 7, 100} { + var seen []uuid.UUID + var after *uuid.UUID + for { + page, err := f.messages.ListByChat(f.ctx, c.ID, after, limit) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + if len(page) > limit { + t.Fatalf("limit %d: page has %d items", limit, len(page)) + } + for _, m := range page { + seen = append(seen, m.ID) + } + last := page[len(page)-1].ID + after = &last + } + if len(seen) != total { + t.Fatalf("limit %d: walked %d messages, want %d", limit, len(seen), total) + } + for i := range seen { + if seen[i] != expected[i] { + t.Fatalf("limit %d: position %d has %s, want %s", limit, i, seen[i], expected[i]) + } + } + } +} + +func TestAfterUnknownIDStartsAtNextGreater(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + a := f.newMessage(t, c.ID, alice) + gap := ident.New() // never stored, but sorts between a and b + b := f.newMessage(t, c.ID, alice) + for _, m := range []chat.Message{a, b} { + if err := f.appendErr(m); err != nil { + t.Fatal(err) + } + } + got, err := f.messages.ListByChat(f.ctx, c.ID, &gap, 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != b.ID { + t.Fatalf("want only b after the gap id, got %v", got) + } +} + +func TestConcurrentAppendsKeepEveryMessage(t *testing.T) { + f := newFixture(t) + alice := f.mustUser(t) + c := f.mustChat(t, alice) + const clients, perClient = 10, 10 + var wg sync.WaitGroup + for range clients { + wg.Go(func() { + for range perClient { + if err := f.appendErr(f.newMessage(t, c.ID, alice)); err != nil { + t.Error(err) + } + } + }) + } + wg.Wait() + got, err := f.messages.ListByChat(f.ctx, c.ID, nil, clients*perClient+1) + if err != nil { + t.Fatal(err) + } + if len(got) != clients*perClient { + t.Fatalf("stored %d messages, want %d", len(got), clients*perClient) + } + for i := 1; i < len(got); i++ { + if bytes.Compare(got[i-1].ID[:], got[i].ID[:]) >= 0 { + t.Fatalf("history out of order at %d", i) + } + } +} diff --git a/internal/health/handler.go b/internal/health/handler.go new file mode 100644 index 00000000..9b14e880 --- /dev/null +++ b/internal/health/handler.go @@ -0,0 +1,77 @@ +// Package health serves the probes. Liveness only proves the process +// answers. Readiness means the process can serve the synchronous API: it is +// switched off during shutdown, so an orchestrator stops routing new traffic +// before the listener closes, and it fails while a critical dependency is +// unreachable. +package health + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +// checkTimeout is shared by all checks of one probe, so the probe answers +// within it even when every dependency hangs; the Docker healthcheck allows +// two seconds. +const checkTimeout = time.Second + +// Check is a dependency the API cannot serve requests without. +type Check struct { + Name string + Probe func(ctx context.Context) error +} + +type Handler struct { + ready atomic.Bool + checks []Check +} + +type status struct { + Status string `json:"status"` +} + +func New(checks ...Check) *Handler { + h := &Handler{checks: checks} + h.ready.Store(true) + return h +} + +func (h *Handler) SetReady(ready bool) { + h.ready.Store(ready) +} + +func (h *Handler) IsReady() bool { + return h.ready.Load() +} + +func (h *Handler) Register(r gin.IRouter) { + r.GET("/health/live", h.live) + r.GET("/health/ready", h.readyz) +} + +func (h *Handler) live(c *gin.Context) { + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} + +func (h *Handler) readyz(c *gin.Context) { + if !h.ready.Load() { + httpx.Error(c, apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(c.Request.Context(), checkTimeout) + defer cancel() + for _, check := range h.checks { + if err := check.Probe(ctx); err != nil { + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, check.Name+" is unavailable", err)) + return + } + } + httpx.JSON(c, http.StatusOK, status{Status: "ok"}) +} diff --git a/internal/imaging/large_png.go b/internal/imaging/large_png.go new file mode 100644 index 00000000..e5d468bd --- /dev/null +++ b/internal/imaging/large_png.go @@ -0,0 +1,96 @@ +package imaging + +import ( + "bytes" + "encoding/binary" + "image" + "image/color" + "image/draw" + "image/png" + "io" + "os" +) + +// PrepareLargePNG runs in the worker's isolated codec child. ImageMagick's +// working scanline for a 40M-by-1 RGBA64 PNG exceeds the entire worker cgroup. +// Go can decode that source in the same memory budget as the upload validator. +// Only a reduced preview and the original centre square cross into ImageMagick. +func PrepareLargePNG(r io.Reader) error { + data, err := io.ReadAll(io.LimitReader(r, MaxUploadBytes+1)) + if err != nil { + return err + } + cfg, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil || int64(len(data)) > MaxUploadBytes || max(cfg.Width, cfg.Height) <= maxMagickPNGAxis || + cfg.Height <= 0 || cfg.Width <= 0 || int64(cfg.Width) > MaxImagePixels/int64(cfg.Height) { + return ErrInvalidImage + } + src, err := png.Decode(bytes.NewReader(data)) + if err != nil { + return ErrInvalidImage + } + // The Go PNG decoder returns standard-library images, all with RGBA64At. + pixels := src.(image.RGBA64Image) + size := fitSize(cfg, 1280) + preview := image.NewRGBA64(image.Rectangle{Max: size}) + for y := range size.Y { + y0 := int(int64(y) * int64(cfg.Height) / int64(size.Y)) + y1 := int(int64(y+1) * int64(cfg.Height) / int64(size.Y)) + for x := range size.X { + x0 := int(int64(x) * int64(cfg.Width) / int64(size.X)) + x1 := int(int64(x+1) * int64(cfg.Width) / int64(size.X)) + preview.SetRGBA64(x, y, meanColor(pixels, image.Rect(x0, y0, x1, y1))) + } + } + // A source wider or taller than 65535 and limited to 40 MP has a centre + // square of at most 610 pixels. Copy from the full-resolution original. + side := min(cfg.Width, cfg.Height) + center := image.NewRGBA64(image.Rect(0, 0, side, side)) + draw.Draw(center, center.Bounds(), src, image.Pt((cfg.Width-side)/2, (cfg.Height-side)/2), draw.Src) + exif := pngEXIF(data) + if err := writePNG("preview.png", preview, exif); err != nil { + return err + } + return writePNG("center.png", center, exif) +} + +func meanColor(src image.RGBA64Image, box image.Rectangle) color.RGBA64 { + var r, g, b, a uint64 + for y := box.Min.Y; y < box.Max.Y; y++ { + for x := box.Min.X; x < box.Max.X; x++ { + c := src.RGBA64At(x, y) + r += uint64(c.R) + g += uint64(c.G) + b += uint64(c.B) + a += uint64(c.A) + } + } + n := uint64(box.Dx()) * uint64(box.Dy()) + return color.RGBA64{R: uint16(r / n), G: uint16(g / n), B: uint16(b / n), A: uint16(a / n)} +} + +// Preserve the original eXIf chunk through the private intermediate files. +// The final encoder applies orientation and strips the metadata. +func pngEXIF(data []byte) []byte { + for chunks := data[8:]; len(chunks) >= 12; { + size := int64(binary.BigEndian.Uint32(chunks[:4])) + 12 + if size > int64(len(chunks)) { + break + } + if string(chunks[4:8]) == "eXIf" { + return chunks[:size] + } + chunks = chunks[size:] + } + return nil +} + +func writePNG(path string, img image.Image, exif []byte) error { + var data bytes.Buffer + if err := png.Encode(&data, img); err != nil { + return err + } + // Insert eXIf immediately after the 8-byte signature and 25-byte IHDR. + content := data.Bytes() + return os.WriteFile(path, bytes.Join([][]byte{content[:33], exif, content[33:]}, nil), 0600) +} diff --git a/internal/imaging/render.go b/internal/imaging/render.go new file mode 100644 index 00000000..72b38ef3 --- /dev/null +++ b/internal/imaging/render.go @@ -0,0 +1,186 @@ +package imaging + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + "golang.org/x/sync/semaphore" +) + +const ( + MaxUploadBytes int64 = 10 << 20 + MaxImagePixels int64 = 40_000_000 + // This bounds a 16-bit RGBA scanline on the ordinary codec path to 512 KiB. + maxMagickPNGAxis = 65_535 +) + +type Variant struct { + Name, MIMEType string + Width, Height int + Data []byte +} + +var ErrInvalidImage = errors.New("invalid image or pipeline version") + +// Reserve both slots for a PNG whose full-width decoder scanlines can each +// occupy hundreds of MiB. A worker with a 2 GiB cgroup can run two ordinary jobs. +var decoderSlots = semaphore.NewWeighted(2) + +func scratchRoot() string { + return filepath.Join(os.TempDir(), "messenger-worker-imaging") +} + +// CleanScratch reclaims pixel caches left by a killed worker. The consumer +// calls it before starting any jobs in its container's private filesystem. +func CleanScratch() error { + return os.RemoveAll(scratchRoot()) +} + +func newRenderDir() (string, error) { + root := scratchRoot() + if err := os.MkdirAll(root, 0700); err != nil { + return "", err + } + return os.MkdirTemp(root, "attachment-") +} + +// Render decodes once in a deadline-bound child process. Its pixel cache can +// spill to a private disk directory when a high-resolution source exceeds RAM. +func Render(ctx context.Context, source []byte, version int) ([]Variant, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if int64(len(source)) > MaxUploadBytes || (version != 1 && version != 2) { + return nil, ErrInvalidImage + } + cfg, format, err := image.DecodeConfig(bytes.NewReader(source)) + if err != nil || (format != "jpeg" && format != "png") || + cfg.Height < 1 || cfg.Width < 1 || int64(cfg.Width) > MaxImagePixels/int64(cfg.Height) { + return nil, ErrInvalidImage + } + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + widePNG := format == "png" && max(cfg.Width, cfg.Height) > maxMagickPNGAxis + weight := int64(1) + if widePNG { + weight = 2 + } + if err := decoderSlots.Acquire(ctx, weight); err != nil { + return nil, err + } + defer decoderSlots.Release(weight) + dir, err := newRenderDir() + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + input := format + ":-[0]" + if widePNG { + executable, err := os.Executable() + if err != nil { + return nil, err + } + cmd := exec.CommandContext(ctx, executable, "--prepare-large-png") + cmd.Env = append(os.Environ(), "GOMAXPROCS=1", "GOMEMLIMIT=1280MiB") + cmd.Dir = dir + cmd.Stdin = bytes.NewReader(source) + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("reduce large PNG: %w", err) + } + input = "png:preview.png" + } + + output := []Variant{ + {Name: "thumbnail", MIMEType: "image/jpeg"}, + {Name: "crop", MIMEType: "image/jpeg"}, + {Name: "optimized", MIMEType: "image/webp"}, + } + args := []string{ + "-limit", "thread", "1", "-limit", "memory", "96MiB", + "-limit", "map", "0", "-limit", "disk", "2GiB", + input, "+repage", "-colorspace", "sRGB", + } + for _, v := range output { + args = append(args, "(") + if widePNG && v.Name == "crop" { + args = append(args, "png:center.png") + } else { + args = append(args, "+clone") + } + quality := 85 + switch v.Name { + case "thumbnail": + args = append(args, "-resize", fit(cfg, 256), "-auto-orient") + case "crop": + side := min(cfg.Width, cfg.Height) + square := fmt.Sprintf("%dx%d+0+0", side, side) + args = append(args, "-auto-orient", "-gravity", "center", "-crop", square, "+repage", "-resize", "128x128!") + case "optimized": + quality = 80 + if version == 2 { + quality = 64 + } + args = append(args, "-resize", fit(cfg, 1280), "-auto-orient") + } + encoder := "webp" + if v.MIMEType == "image/jpeg" { + encoder = "jpeg" + args = append(args, "-background", "white", "-alpha", "remove", "-alpha", "off") + } + args = append(args, "-strip", "-quality", strconv.Itoa(quality), "-write", encoder+":"+v.Name, + "-format", "%w %h\n", "-write", "info:", "+delete", ")") + } + cmd := exec.CommandContext(ctx, "magick", append(args, "null:")...) + cmd.Dir = dir + cmd.Env = append(cmd.Environ(), "MAGICK_TEMPORARY_PATH="+dir) + if !widePNG { + cmd.Stdin = bytes.NewReader(source) + } + dimensions, err := cmd.Output() + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("render image variants: %w", err) + } + sizes := bytes.NewReader(dimensions) + for i := range output { + v := &output[i] + if _, err := fmt.Fscan(sizes, &v.Width, &v.Height); err != nil { + return nil, fmt.Errorf("read %s dimensions: %w", v.Name, err) + } + v.Data, err = os.ReadFile(filepath.Join(dir, v.Name)) + if err != nil { + return nil, err + } + } + return output, nil +} + +// Compute geometry before auto-orientation. Rotation exchanges the axes after +// resizing; bounding by a square gives the same floor-rounded dimensions. +func fit(cfg image.Config, limit int) string { + size := fitSize(cfg, limit) + return fmt.Sprintf("%dx%d!", size.X, size.Y) +} + +func fitSize(cfg image.Config, limit int) image.Point { + longest := max(cfg.Width, cfg.Height) + scale := int64(min(longest, limit)) + width := max(1, int64(cfg.Width)*scale/int64(longest)) + height := max(1, int64(cfg.Height)*scale/int64(longest)) + return image.Pt(int(width), int(height)) +} diff --git a/internal/imaging/render_test.go b/internal/imaging/render_test.go new file mode 100644 index 00000000..d15d2875 --- /dev/null +++ b/internal/imaging/render_test.go @@ -0,0 +1,314 @@ +package imaging + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestMain(m *testing.M) { + if len(os.Args) == 2 && os.Args[1] == "--prepare-large-png" { + if err := PrepareLargePNG(os.Stdin); err != nil { + os.Exit(1) + } + return + } + os.Exit(m.Run()) +} + +func TestWidePNGPreservesCenterPixelAndAveragesStripes(t *testing.T) { + requireCodec(t) + img := image.NewNRGBA(image.Rect(0, 0, 70_001, 1)) + for x := range img.Bounds().Dx() { + shade := uint8((x % 2) * 255) + img.SetNRGBA(x, 0, color.NRGBA{shade, shade, shade, 255}) + } + img.SetNRGBA(35_000, 0, color.NRGBA{255, 0, 0, 255}) + var source bytes.Buffer + if err := png.Encode(&source, img); err != nil { + t.Fatal(err) + } + results := variants(t, source.Bytes(), 1) + thumb := decodedJPEG(t, results["thumbnail"], 256, 1) + r, _, _, _ := thumb.At(2, 0).RGBA() + if r < 0x7000 || r > 0x9000 { + t.Fatalf("stripes were not averaged: %x", r) + } + crop := decodedJPEG(t, results["crop"], 128, 128) + r, g, b, _ := crop.At(64, 64).RGBA() + if r < 0xe000 || max(g, b) > 0x2000 { + t.Fatalf("center crop used reduced image: %x/%x/%x", r, g, b) + } +} + +func TestWorkerPNGChildStartsWithoutServiceConfig(t *testing.T) { + if os.Getenv("CODEC_REQUIRED") != "1" { + t.Skip("run make codec-test in the packaged worker runtime") + } + cmd := exec.CommandContext(t.Context(), "worker", "--prepare-large-png") + cmd.Dir = t.TempDir() + cmd.Stdin = bytes.NewReader(testPicture(t, 70_000, 1)) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("worker PNG child: %v, %s", err, output) + } + for name, wantWidth := range map[string]int{"preview.png": 1280, "center.png": 1} { + data, err := os.ReadFile(filepath.Join(cmd.Dir, name)) + if err != nil { + t.Fatal(err) + } + cfg, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width != wantWidth || cfg.Height != 1 { + t.Fatalf("%s %dx%d: %v", name, cfg.Width, cfg.Height, err) + } + } +} + +func testPicture(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + // Transparent margins, fine detail in the centre to distinguish WebP profiles. + if x >= width/4 && x < width-width/4 { + img.SetNRGBA(x, y, color.NRGBA{uint8(x * 29), uint8(y * 7), uint8(x ^ y), 255}) + } + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestRejectInvalidSources(t *testing.T) { + huge := testPicture(t, 2, 2) + binary.BigEndian.PutUint32(huge[16:20], 20_000_001) + binary.BigEndian.PutUint32(huge[29:33], crc32.ChecksumIEEE(huge[12:29])) + for _, tc := range []struct { + name string + data []byte + version int + }{ + {"unrecognized image", []byte("hello"), 1}, + {"oversize body", make([]byte, (10<<20)+1), 1}, + {"oversize pixel count", huge, 1}, + {"unsupported version", testPicture(t, 2, 2), 3}, + } { + t.Run(tc.name, func(t *testing.T) { + results, err := Render(context.Background(), tc.data, tc.version) + if !errors.Is(err, ErrInvalidImage) || len(results) != 0 { + t.Fatalf("got %d variants, err %v", len(results), err) + } + }) + } +} + +func requireCodec(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("magick"); err != nil { + if os.Getenv("CODEC_REQUIRED") == "1" { + t.Fatal(err) + } + t.Skip("ImageMagick 7 with JPEG, PNG and WebP coders required") + } +} + +func TestFortyMegapixelFixtures(t *testing.T) { + dir := os.Getenv("CODEC_FIXTURES") + if dir == "" { + t.Skip("run make codec-test CODEC_STRESS=1 with generated decoder fixtures") + } + requireCodec(t) + for _, tc := range []struct { + name string + thumbnail, optimized image.Point + }{ + {"progressive-rgb-444-40mp.jpg", image.Pt(256, 160), image.Pt(1280, 800)}, + {"progressive-cmyk-444-40mp.jpg", image.Pt(256, 160), image.Pt(1280, 800)}, + {"wide-rgba64-40mp.png", image.Pt(256, 1), image.Pt(1280, 1)}, + } { + t.Run(tc.name, func(t *testing.T) { + source, err := os.ReadFile(filepath.Join(dir, tc.name)) + if err != nil { + t.Fatal(err) + } + out := variants(t, source, 1) + decodedJPEG(t, out["thumbnail"], tc.thumbnail.X, tc.thumbnail.Y) + decodedJPEG(t, out["crop"], 128, 128) + decodedWebP(t, out["optimized"], tc.optimized.X, tc.optimized.Y) + t.Logf("encoded bytes: thumbnail=%d crop=%d optimized=%d", len(out["thumbnail"].Data), len(out["crop"].Data), len(out["optimized"].Data)) + }) + } +} + +func variants(t *testing.T, source []byte, version int) map[string]Variant { + t.Helper() + out, err := Render(context.Background(), source, version) + if err != nil { + t.Fatal(err) + } + if len(out) != 3 { + t.Fatalf("expected 3 variants, got %d", len(out)) + } + result := make(map[string]Variant) + for _, v := range out { + result[v.Name] = v + } + return result +} + +func decodedJPEG(t *testing.T, v Variant, width, height int) image.Image { + t.Helper() + im, err := jpeg.Decode(bytes.NewReader(v.Data)) + if err != nil { + t.Fatal(err) + } + if v.MIMEType != "image/jpeg" || im.Bounds() != image.Rect(0, 0, width, height) || v.Width != width || v.Height != height { + t.Fatalf("JPEG metadata %s %dx%d, actual %v, want %dx%d", v.MIMEType, v.Width, v.Height, im.Bounds(), width, height) + } + return im +} + +func decodedWebP(t *testing.T, v Variant, width, height int) { + t.Helper() + if v.MIMEType != "image/webp" || v.Width != width || v.Height != height { + t.Fatalf("invalid optimized metadata %s %dx%d", v.MIMEType, v.Width, v.Height) + } + probe := exec.Command("magick", "webp:-", "-format", "%wx%h", "info:") + probe.Stdin = bytes.NewReader(v.Data) + got, err := probe.CombinedOutput() + if err != nil || string(got) != fmt.Sprintf("%dx%d", width, height) { + t.Fatalf("decode optimized: %s, %v", got, err) + } +} + +func TestRenderProfiles(t *testing.T) { + requireCodec(t) + source := testPicture(t, 1301, 707) + v1 := variants(t, source, 1) + thumb := decodedJPEG(t, v1["thumbnail"], 256, 139) + r, g, b, _ := thumb.At(7, 5).RGBA() + if min(r, g, b) < 0xf000 { + t.Fatalf("transparent thumbnail margin became %x/%x/%x", r, g, b) + } + crop := decodedJPEG(t, v1["crop"], 128, 128) + r, g, b, _ = crop.At(13, 64).RGBA() + if min(r, g, b) > 0xe000 { + t.Error("crop retained the outer transparent margin") + } + webp := v1["optimized"] + decodedWebP(t, webp, 1280, 695) + if bytes.Equal(webp.Data, variants(t, source, 2)["optimized"].Data) { + t.Error("two WebP profiles gave identical bytes on a detailed image") + } + if !bytes.Equal(webp.Data, variants(t, source, 1)["optimized"].Data) { + t.Error("same version produced different bytes") + } +} + +func TestTinyRotatedJPEG(t *testing.T) { + requireCodec(t) + img := image.NewRGBA(image.Rect(0, 0, 32, 16)) + for y := range 16 { + for x := range 32 { + if x < 16 { + img.Set(x, y, color.RGBA{255, 0, 0, 255}) + } else { + img.Set(x, y, color.RGBA{0, 0, 255, 255}) + } + } + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, nil); err != nil { + t.Fatal(err) + } + // APP1 with a little-endian TIFF orientation of 6 (rotate 90 degrees clockwise). + exif := []byte("\xff\xe1\x00\x22Exif\x00\x00II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00") + source := bytes.Join([][]byte{buf.Bytes()[:2], exif, buf.Bytes()[2:]}, nil) + v := variants(t, source, 1) + thumb := decodedJPEG(t, v["thumbnail"], 16, 32) + r, _, b, _ := thumb.At(8, 5).RGBA() + if r < 0xc000 || b > 0x3000 { + t.Fatal("thumbnail was not rotated clockwise") + } + decodedJPEG(t, v["crop"], 128, 128) + if v["optimized"].Width != 16 || v["optimized"].Height != 32 { + t.Fatal("optimized tiny image was enlarged or not rotated") + } + for name, variant := range v { + if bytes.Contains(bytes.ToUpper(variant.Data), []byte("EXIF")) { + t.Errorf("%s retained EXIF", name) + } + } +} + +func TestRenderHonorsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Render(ctx, testPicture(t, 2, 2), 1) + if !errors.Is(err, context.Canceled) { + t.Fatalf("got %v", err) + } +} + +func TestOnePixelMinorAxis(t *testing.T) { + requireCodec(t) + v := variants(t, testPicture(t, 1, 4096), 1) + decodedJPEG(t, v["thumbnail"], 1, 256) +} + +func TestCropUsesRasterCenterOnOffsetPNGCanvas(t *testing.T) { + requireCodec(t) + src, err := exec.CommandContext(t.Context(), "magick", "-size", "320x200", "xc:red", + "-fill", "blue", "-draw", "rectangle 0,0 55,199 rectangle 265,0 319,199", + "-repage", "2400x1800+850+30", "png:-").Output() + if err != nil { + t.Fatal(err) + } + crop := decodedJPEG(t, variants(t, src, 1)["crop"], 128, 128) + for _, x := range []int{1, 64, 126} { + r, g, b, _ := crop.At(x, 64).RGBA() + if r < 0xe000 || max(g, b) > 0x2000 { + t.Fatalf("crop pixel (%d,64) = %x/%x/%x, want red", x, r, g, b) + } + } +} + +func TestCleanScratchRemovesInterruptedRender(t *testing.T) { + temp := t.TempDir() + for _, key := range []string{"TMPDIR", "TMP", "TEMP"} { + t.Setenv(key, temp) + } + dir, err := newRenderDir() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "magick-cache"), []byte("pixels"), 0600); err != nil { + t.Fatal(err) + } + unrelated := filepath.Join(temp, "another-process") + if err := os.Mkdir(unrelated, 0700); err != nil { + t.Fatal(err) + } + if err := CleanScratch(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("interrupted render directory still accessible: %v", err) + } + if _, err := os.Stat(unrelated); err != nil { + t.Fatal(err) + } +} diff --git a/internal/pipeline/job.go b/internal/pipeline/job.go new file mode 100644 index 00000000..72618e24 --- /dev/null +++ b/internal/pipeline/job.go @@ -0,0 +1,102 @@ +package pipeline + +import ( + "crypto/sha256" + "errors" + "fmt" + "math/rand/v2" + "regexp" + "time" + + "github.com/google/uuid" +) + +type Job struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +var traceParent = regexp.MustCompile("^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + +func (j Job) Valid() bool { + return j.SchemaVersion == 1 && j.Type == "attachment.process.v1" && + j.JobID != uuid.Nil && j.AttachmentID != uuid.Nil && j.PipelineVersion >= 1 && + j.Attempt >= 1 && !j.CreatedAt.IsZero() && + (j.Traceparent == "" || traceParent.MatchString(j.Traceparent)) && len(j.Tracestate) <= 512 +} + +type Claim struct { + Job + Token uuid.UUID + SourceKey string + Hash []byte + SizeBytes int64 + MIMEType string +} + +type Variant struct { + Name, Key, MIMEType string + Width, Height int + SizeBytes int64 + SHA256 []byte +} + +func VariantKey(id uuid.UUID, version int, name string, hash []byte) string { + return fmt.Sprintf("variants/%s/%d/%s/%x", id, version, name, hash) +} + +func validateVariants(c *Claim, variants []Variant) error { + if len(variants) != 3 { + return errors.New("three variants are required") + } + seen := map[string]bool{} + for _, v := range variants { + mime := "image/jpeg" + switch v.Name { + case "thumbnail", "crop": + case "optimized": + mime = "image/webp" + default: + return errors.New("unknown variant") + } + if seen[v.Name] || v.MIMEType != mime || len(v.SHA256) != sha256.Size || + v.Key != VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, v.SHA256) || + v.Width <= 0 || v.Height <= 0 || v.SizeBytes <= 0 { + return errors.New("invalid variant metadata") + } + seen[v.Name] = true + } + return nil +} + +type RetryPolicy struct { + MaxAttempts int + BaseDelay, MaxDelay time.Duration +} + +func (p RetryPolicy) validate() error { + if p.MaxAttempts < 1 || p.BaseDelay < time.Microsecond || p.MaxDelay < p.BaseDelay { + return errors.New("invalid pipeline retry policy") + } + return nil +} + +// Equal jitter keeps a minimum delay while distributing simultaneous failures. +func (p RetryPolicy) delay(attempt int) time.Duration { + limit := p.BaseDelay + for i := 1; i < attempt && limit < p.MaxDelay; i++ { + if limit > p.MaxDelay/2 { + limit = p.MaxDelay + } else { + limit *= 2 + } + } + return limit/2 + time.Duration(rand.Int64N(int64(limit-limit/2)+1)) +} diff --git a/internal/pipeline/operator.go b/internal/pipeline/operator.go new file mode 100644 index 00000000..be48c696 --- /dev/null +++ b/internal/pipeline/operator.go @@ -0,0 +1,135 @@ +package pipeline + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "messenger/internal/attachment" + "messenger/internal/platform/apperr" +) + +type FailedJob struct { + JobID uuid.UUID `json:"job_id"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + ErrorCode string `json:"error_code"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s *Store) Failed(ctx context.Context) ([]FailedJob, error) { + rows, err := s.pool.Query(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,error_code,updated_at + FROM attachment_jobs WHERE state='failed' ORDER BY updated_at,job_id`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, pgx.RowToStructByPos[FailedJob]) +} + +// Lock the attachment before checking the current target, its source or an S3 +// object. This serializes replay and version changes with source collection. +func lockTarget(ctx context.Context, tx pgx.Tx, id uuid.UUID) (int, []byte, error) { + var version int + var hash []byte + err := tx.QueryRow(ctx, `SELECT target_version,file_hash FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&version, &hash) + if errors.Is(err, pgx.ErrNoRows) { + err = apperr.NotFound("attachment not found") + } + return version, hash, err +} + +func availableSource(ctx context.Context, preferred, fallback string, exists func(context.Context, string) (bool, error)) (string, error) { + ok, err := exists(ctx, preferred) + if err != nil { + return "", err + } + if ok { + return preferred, nil + } + if fallback != preferred { + ok, err = exists(ctx, fallback) + if err != nil { + return "", err + } + if ok { + return fallback, nil + } + } + return "", apperr.Conflict("processing source is missing") +} + +func (s *Store) Reprocess(ctx context.Context, id uuid.UUID, version int, exists func(context.Context, string) (bool, error)) error { + if version != 1 && version != 2 { + return apperr.Invalid("unsupported pipeline version") + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + current, hash, err := lockTarget(ctx, tx, id) + if err != nil { + return err + } + if version <= current { + return apperr.Conflict("target version must increase") + } + var source string + if err := tx.QueryRow(ctx, `SELECT source_key FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 FOR UPDATE`, id, current).Scan(&source); err != nil { + return err + } + source, err = availableSource(ctx, attachment.BlobKey(hash), source, exists) + if err != nil { + return err + } + // Keep committed result rows immutable and readable until the switch. + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp() + WHERE attachment_id=$1 AND pipeline_version=$2 AND state<>'ready'`, id, current); err != nil { + return err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key) + VALUES ($1,$2,$3)`, id, version, source); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET target_version=$2 WHERE id=$1`, id, version) + return err + }) +} + +func (s *Store) Replay(ctx context.Context, id uuid.UUID, actor string, exists func(context.Context, string) (bool, error)) error { + actor = strings.TrimSpace(actor) + if actor == "" || len(actor) > 128 { + return apperr.Invalid("replay requires an operator name of up to 128 bytes") + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + current, hash, err := lockTarget(ctx, tx, id) + if err != nil { + return err + } + var jobID uuid.UUID + var source, code string + var attempt int + err = tx.QueryRow(ctx, `SELECT job_id,source_key,error_code,attempt FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 AND state='failed' FOR UPDATE`, id, current). + Scan(&jobID, &source, &code, &attempt) + if errors.Is(err, pgx.ErrNoRows) { + return apperr.Conflict("current job is not failed") + } + if err != nil { + return err + } + source, err = availableSource(ctx, source, attachment.BlobKey(hash), exists) + if err != nil { + return err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,attempt,actor,previous_error) VALUES ($1,$2,$3,$4)`, + jobID, attempt+1, actor, code); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempt=attempt+1,budget_attempt=1,source_key=$2, + published_at=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),error_code=NULL WHERE job_id=$1`, jobID, source) + return err + }) +} diff --git a/internal/pipeline/retry.go b/internal/pipeline/retry.go new file mode 100644 index 00000000..196a4389 --- /dev/null +++ b/internal/pipeline/retry.go @@ -0,0 +1,86 @@ +package pipeline + +import ( + "context" + "errors" + "regexp" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$") + +type Failure struct { + Code string + Permanent bool +} + +type leaseState bool + +const ( + activeLease leaseState = false + expiredLease leaseState = true +) + +func (s *Store) Fail(ctx context.Context, c *Claim, failure Failure, policy RetryPolicy) (bool, error) { + if err := policy.validate(); err != nil { + return false, err + } + if !errorCodePattern.MatchString(failure.Code) { + return false, errors.New("invalid failure code") + } + var changed bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var budget int + err := tx.QueryRow(ctx, `SELECT budget_attempt FROM attachment_jobs + WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 AND attempt=$4 + AND state='processing' AND lease_token=$5 AND lease_until>clock_timestamp() FOR UPDATE`, + c.JobID, c.AttachmentID, c.PipelineVersion, c.Attempt, c.Token).Scan(&budget) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + changed, err = finishAttempt(ctx, tx, c.JobID, budget, failure, activeLease, policy) + return err + }) + return changed && err == nil, err +} + +func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error) { + if err := policy.validate(); err != nil { + return false, err + } + var changed bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var id uuid.UUID + var budget int + err := tx.QueryRow(ctx, `SELECT job_id,budget_attempt FROM attachment_jobs + WHERE state='processing' AND lease_until<=clock_timestamp() + ORDER BY lease_until,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id, &budget) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + changed, err = finishAttempt(ctx, tx, id, budget, Failure{Code: "lease_expired"}, expiredLease, policy) + return err + }) + return changed && err == nil, err +} + +// The caller holds the job row. Only the recovery path may advance an expired +// attempt. A worker whose lease elapsed during its SQL call has lost ownership. +func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, failure Failure, lease leaseState, p RetryPolicy) (bool, error) { + retry := !failure.Permanent && budget < p.MaxAttempts + tag, err := tx.Exec(ctx, `UPDATE attachment_jobs + SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END, + attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int, + available_at=clock_timestamp()+$4::bigint*interval '1 microsecond', + published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp() + WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, failure.Code, retry, p.delay(budget).Microseconds(), bool(lease)) + return tag.RowsAffected() == 1, err +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go new file mode 100644 index 00000000..da9ad4eb --- /dev/null +++ b/internal/pipeline/store.go @@ -0,0 +1,143 @@ +package pipeline + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" +) + +type Store struct{ pool *pgxpool.Pool } + +func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +// The callback waits for a routed publisher confirm with a finite timeout. +// Losing that confirm or the SQL commit leaves the same envelope deliverable. +// NO KEY UPDATE excludes other publishers but allows messages to link a queued attachment. +func (s *Store) PublishOne(ctx context.Context, publish func(context.Context, Job) error) (bool, error) { + var found bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + j := Job{SchemaVersion: 1, Type: "attachment.process.v1"} + err := tx.QueryRow(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,created_at + FROM attachment_jobs WHERE state='queued' AND published_at IS NULL AND available_at<=clock_timestamp() + ORDER BY available_at,job_id LIMIT 1 FOR NO KEY UPDATE SKIP LOCKED`). + Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + found = true + if err := publish(ctx, j); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET published_at=clock_timestamp() + WHERE job_id=$1 AND attempt=$2 AND state='queued'`, j.JobID, j.Attempt) + return err + }) + return found, err +} + +// Claim checks the envelope against SQL and sources all object metadata from SQL. +// A processing duplicate can be acknowledged: lease recovery is independent +// of broker redelivery and persists a fresh outbox intent after worker death. +func (s *Store) Claim(ctx context.Context, j Job, lease time.Duration) (*Claim, error) { + if !j.Valid() || lease < time.Microsecond { + return nil, errors.New("invalid pipeline job or lease") + } + c := Claim{Job: j, Token: uuid.New()} + err := s.pool.QueryRow(ctx, `UPDATE attachment_jobs j SET state='processing', lease_token=$6, + lease_until=clock_timestamp()+$7::bigint*interval '1 microsecond', updated_at=clock_timestamp() + FROM attachments a JOIN files f ON f.hash=a.file_hash + WHERE j.job_id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.attempt=$4 AND j.created_at=$5 + AND a.id=j.attachment_id AND a.target_version=j.pipeline_version + AND j.state='queued' AND j.available_at<=clock_timestamp() + RETURNING j.source_key,f.hash,f.size_bytes,f.mime_type`, + j.JobID, j.AttachmentID, j.PipelineVersion, j.Attempt, j.CreatedAt, c.Token, lease.Microseconds()). + Scan(&c.SourceKey, &c.Hash, &c.SizeBytes, &c.MIMEType) + if err == nil { + return &c, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, err + } + var incomplete bool + err = s.pool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE j.job_id=$1 AND j.state='ready' AND + (coalesce(a.published_version,0)( + SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id AND v.pipeline_version=j.pipeline_version)) + )`, j.JobID).Scan(&incomplete) + if err != nil { + return nil, err + } + if incomplete { + return nil, errors.New("ready job has incomplete durable result") + } + return nil, nil +} + +var errLostClaim = errors.New("claim expired or superseded") + +func (s *Store) Complete(ctx context.Context, c *Claim, variants []Variant) (bool, error) { + if err := validateVariants(c, variants); err != nil { + return false, err + } + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if err := attachment.LockHash(ctx, tx, c.Hash); err != nil { + return err + } + // Lock the attachment before the job, in the same order as operator commands. + tag, err := tx.Exec(ctx, `SELECT 1 FROM attachments WHERE id=$1 AND target_version=$2 FOR UPDATE`, c.AttachmentID, c.PipelineVersion) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + tag, err = tx.Exec(ctx, `SELECT 1 FROM attachment_jobs WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 + AND state='processing' AND lease_token=$4 AND lease_until>clock_timestamp() FOR UPDATE`, + c.JobID, c.AttachmentID, c.PipelineVersion, c.Token) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + for _, v := range variants { + _, err := tx.Exec(ctx, `INSERT INTO attachment_variants + (attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, + c.AttachmentID, c.PipelineVersion, v.Name, v.Key, v.MIMEType, v.Width, v.Height, v.SizeBytes, v.SHA256) + if err != nil { + return err + } + } + if _, err = tx.Exec(ctx, `UPDATE files SET status='ready' WHERE hash=(SELECT file_hash FROM attachments WHERE id=$1)`, c.AttachmentID); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET published_version=$2 WHERE id=$1`, c.AttachmentID, c.PipelineVersion); err != nil { + return err + } + // Recheck DB time after every operation that could have waited on a lock. + tag, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='ready',lease_until=NULL,lease_token=NULL, + error_code=NULL,updated_at=clock_timestamp() WHERE job_id=$1 AND lease_until>clock_timestamp()`, c.JobID) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + return nil + }) + if errors.Is(err, errLostClaim) { + return false, nil + } + return err == nil, err +} diff --git a/internal/pipeline/store_test.go b/internal/pipeline/store_test.go new file mode 100644 index 00000000..234e1691 --- /dev/null +++ b/internal/pipeline/store_test.go @@ -0,0 +1,334 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + + "messenger/internal/testenv" + "messenger/migrations" +) + +func TestPipelineTransitions(t *testing.T) { + ctx := t.Context() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(db.Close) + s := NewStore(db.Pool) + exists := func(context.Context, string) (bool, error) { return true, nil } + policy := RetryPolicy{MaxAttempts: 2, BaseDelay: time.Millisecond, MaxDelay: time.Second} + exec := func(t *testing.T, query string, args ...any) { + t.Helper() + if _, err := db.Pool.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + newJob := func(t *testing.T) { + t.Helper() + db.Reset(t) + id := uuid.New() + exec(t, `WITH u AS (INSERT INTO users VALUES (gen_random_uuid(),'test','Test User','hash',now()) RETURNING id), + c AS (INSERT INTO chats VALUES (gen_random_uuid(),'chat',now()) RETURNING id), + f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),10,'image/png','pending') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + SELECT $1,c.id,u.id,'x.png',f.hash,now() FROM u,c,f`, id) + exec(t, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, "staging/"+id.String()) + } + publish := func(t *testing.T) Job { + t.Helper() + var j Job + ok, err := s.PublishOne(ctx, func(_ context.Context, v Job) error { j = v; return nil }) + if err != nil || !ok || !j.Valid() { + t.Fatalf("publish: %+v, %v, %v", j, ok, err) + } + return j + } + claim := func(t *testing.T, j Job) *Claim { + t.Helper() + c, err := s.Claim(ctx, j, time.Minute) + if err != nil || c == nil { + t.Fatalf("claim: %v %v", c, err) + } + return c + } + variants := func(c *Claim) []Variant { + var vv []Variant + for i, name := range []string{"thumbnail", "crop", "optimized"} { + hash := bytes.Repeat([]byte{byte(c.PipelineVersion*10 + i)}, 32) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + vv = append(vv, Variant{Name: name, Key: fmt.Sprintf("variants/%s/%d/%s/%x", c.AttachmentID, c.PipelineVersion, name, hash), Width: 128, Height: 128, SizeBytes: 4, MIMEType: mime, SHA256: hash}) + } + return vv + } + t.Run("unconfirmed publication is retried and concurrent publishers skip locked work", func(t *testing.T) { + newJob(t) + var seen Job + confirmLost := errors.New("confirm lost") + _, err := s.PublishOne(ctx, func(_ context.Context, j Job) error { + seen = j + ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("double publish") }) + if err != nil || ok { + t.Fatalf("second publisher %v %v", ok, err) + } + return confirmLost + }) + if !errors.Is(err, confirmLost) { + t.Fatal(err) + } + j := publish(t) + if j != seen { + t.Fatalf("redelivery changed envelope: %+v %+v", seen, j) + } + ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("duplicate") }) + if ok || err != nil { + t.Fatalf("confirmed work republished: %v %v", ok, err) + } + }) + t.Run("backoff gates publication and consumption of the next attempt", func(t *testing.T) { + newJob(t) + first := claim(t, publish(t)) + policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second} + if ok, err := s.Fail(ctx, first, Failure{Code: "storage_timeout"}, policy); !ok || err != nil { + t.Fatalf("failed %v %v", ok, err) + } + var seconds float64 + if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil { + t.Fatal(err) + } + if seconds < 1.99 || seconds > 4.01 { + t.Fatalf("first retry delay %.3f seconds", seconds) + } + next := first.Job + next.Attempt++ + if c, err := s.Claim(ctx, next, time.Minute); err != nil || c != nil { + t.Fatalf("premature claim %v %v", c, err) + } + if ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("premature publish") }); err != nil || ok { + t.Fatalf("premature publish %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + second := claim(t, publish(t)) + if ok, err := s.Fail(ctx, second, Failure{Code: "storage_timeout"}, policy); !ok || err != nil { + t.Fatalf("second failure %v %v", ok, err) + } + if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil { + t.Fatal(err) + } + if seconds < 2.49 || seconds > 5.01 { + t.Fatalf("capped second retry delay %.3f seconds", seconds) + } + }) + t.Run("parallel duplicates and version change before commit", func(t *testing.T) { + newJob(t) + j := publish(t) + var wg sync.WaitGroup + claims := make(chan *Claim, 8) + for range 8 { + wg.Go(func() { + c, err := s.Claim(ctx, j, time.Minute) + if err != nil { + t.Error(err) + } + if c != nil { + claims <- c + } + }) + } + wg.Wait() + close(claims) + if len(claims) != 1 { + t.Fatalf("%d workers claimed one job", len(claims)) + } + old := <-claims + if old.SourceKey != "staging/"+j.AttachmentID.String() || old.SizeBytes != 10 || !bytes.Equal(old.Hash, bytes.Repeat([]byte{0xab}, 32)) { + t.Fatalf("source metadata: %+v", old) + } + stageOnly := func(_ context.Context, key string) (bool, error) { return key == old.SourceKey, nil } + // A different attachment can have published the same digest. That row's + // status cannot prove the object is still there when the target changes. + exec(t, `UPDATE files SET status='ready'`) + if err := s.Reprocess(ctx, j.AttachmentID, 2, stageOnly); err != nil { + t.Fatal(err) + } + current := claim(t, publish(t)) + if current.SourceKey != old.SourceKey { + t.Fatalf("available staging ignored in favor of missing CAS: %s", current.SourceKey) + } + results := variants(current) + if _, err := s.Complete(ctx, current, results[:2]); err == nil { + t.Fatal("published incomplete result") + } + if ok, err := s.Complete(ctx, current, results); err != nil || !ok { + t.Fatalf("complete %v %v", ok, err) + } + if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok { + t.Fatalf("stale commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, old, Failure{Code: "source_missing", Permanent: true}, policy); err != nil || ok { + t.Fatalf("stale failure %v %v", ok, err) + } + for _, dup := range []Job{current.Job, j, current.Job} { + if c, err := s.Claim(ctx, dup, time.Minute); err != nil || c != nil { + t.Fatalf("duplicate %v %v", c, err) + } + } + var version, count int + var digest []byte + if err := db.Pool.QueryRow(ctx, `SELECT a.published_version, + (SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id), + (SELECT sha256 FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2 AND variant='optimized') + FROM attachments a WHERE a.id=$1`, j.AttachmentID).Scan(&version, &count, &digest); err != nil { + t.Fatal(err) + } + if version != 2 || count != 3 || !bytes.Equal(digest, results[2].SHA256) { + t.Fatalf("published v%d count %d hash %x", version, count, digest) + } + exec(t, `DELETE FROM attachment_variants WHERE variant='crop'`) + if _, err := s.Claim(ctx, current.Job, time.Minute); err == nil { + t.Fatal("ready duplicate acknowledged without complete result") + } + }) + t.Run("dead lease and retry exhaustion then audited replay", func(t *testing.T) { + newJob(t) + first := claim(t, publish(t)) + exec(t, `UPDATE attachment_jobs SET lease_until=clock_timestamp()-interval '1 second'`) + if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { + t.Fatalf("expired commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, first, Failure{Code: "backend_unavailable"}, policy); err != nil || ok { + t.Fatalf("expired fail %v %v", ok, err) + } + if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok { + t.Fatalf("recover %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + second := claim(t, publish(t)) + if first.JobID != second.JobID || first.Attempt != 1 || second.Attempt != 2 || first.Token == second.Token { + t.Fatalf("attempt identity %+v %+v", first, second) + } + if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { + t.Fatalf("lost lease commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, second, Failure{Code: "backend_unavailable"}, policy); err != nil || !ok { + t.Fatalf("second error %v %v", ok, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 { + t.Fatalf("failed %v %v", failed, err) + } + absent := func(context.Context, string) (bool, error) { return false, nil } + if err := s.Replay(ctx, first.AttachmentID, "test", absent); err == nil { + t.Fatal("replayed without source") + } + hasStage := func(_ context.Context, k string) (bool, error) { return k == first.SourceKey, nil } + if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err != nil { + t.Fatal(err) + } + if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err == nil { + t.Fatal("replayed active job") + } + if c, err := s.Claim(ctx, first.Job, time.Minute); c != nil || err != nil { + t.Fatalf("old delivery %v %v", c, err) + } + replay := claim(t, publish(t)) + if replay.Attempt != 3 || replay.JobID != first.JobID { + t.Fatalf("replay identity %+v", replay) + } + if ok, err := s.Fail(ctx, replay, Failure{Code: "backend_unavailable"}, policy); !ok || err != nil { + t.Fatalf("fresh retry budget %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + last := claim(t, publish(t)) + if ok, err := s.Fail(ctx, last, Failure{Code: "source_missing", Permanent: true}, policy); !ok || err != nil { + t.Fatalf("permanent fail %v %v", ok, err) + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachment_replays WHERE actor='operator' AND previous_error='backend_unavailable'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("replay audit %d %v", n, err) + } + }) + t.Run("the previous version remains readable until the entire new result commits", func(t *testing.T) { + newJob(t) + old := claim(t, publish(t)) + if ok, err := s.Complete(ctx, old, variants(old)); err != nil || !ok { + t.Fatalf("first publication %v %v", ok, err) + } + if err := s.Reprocess(ctx, old.AttachmentID, 2, exists); err != nil { + t.Fatal(err) + } + next := claim(t, publish(t)) + if next.SourceKey != fmt.Sprintf("blobs/%x", old.Hash) { + t.Fatalf("version two source %s", next.SourceKey) + } + // Force an actual SQL failure on the second INSERT, after the first one succeeded. + exec(t, `ALTER TABLE attachment_variants ADD CONSTRAINT experiment_failure CHECK (pipeline_version<>2 OR variant<>'crop')`) + if _, err := s.Complete(ctx, next, variants(next)); err == nil { + t.Fatal("expected SQL failure on second variant") + } + exec(t, `ALTER TABLE attachment_variants DROP CONSTRAINT experiment_failure`) + var version, resultCount int + var status string + if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,s.status, + (SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2) + FROM attachment_states s JOIN attachments a USING(id)`).Scan(&version, &status, &resultCount); err != nil { + t.Fatal(err) + } + if version != 1 || status != "ready" || resultCount != 0 { + t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount) + } + if ok, err := s.Fail(ctx, next, Failure{Code: "invalid_image", Permanent: true}, policy); err != nil || !ok { + t.Fatalf("permanent error %v %v", ok, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 || failed[0].Attempt != 1 || failed[0].ErrorCode != "invalid_image" { + t.Fatalf("permanent error not terminal %v %v", failed, err) + } + if err := db.Pool.QueryRow(ctx, `SELECT status FROM attachment_states`).Scan(&status); err != nil || status != "ready" { + t.Fatalf("failed reprocess hid original %s %v", status, err) + } + }) + t.Run("migration preserves Lab4 message links and requeues known originals", func(t *testing.T) { + newJob(t) + provider, err := goose.NewProvider(goose.DialectPostgres, stdlib.OpenDBFromPool(db.Pool), migrations.FS) + if err != nil { + t.Fatal(err) + } + defer provider.Close() + if _, err := provider.DownTo(ctx, 3); err != nil { + t.Fatal(err) + } + exec(t, `UPDATE files SET status='ready'`) + exec(t, `WITH f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('cd',32),'hex'),10,'image/jpeg','failed') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),chat_id,owner_id,'missing.jpg',f.hash,now() FROM attachments,f`) + exec(t, `WITH m AS (INSERT INTO messages(chat_id,id,sender_id,text,created_at) SELECT chat_id,gen_random_uuid(),owner_id,'old',now() FROM attachments LIMIT 1 RETURNING chat_id,id) + INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) SELECT m.chat_id,m.id,a.id,0 FROM m,attachments a WHERE a.filename='x.png'`) + if _, err := provider.Up(ctx); err != nil { + t.Fatal(err) + } + job := claim(t, publish(t)) + if job.SourceKey != fmt.Sprintf("blobs/%x", bytes.Repeat([]byte{0xab}, 32)) { + t.Fatalf("migrated source %s", job.SourceKey) + } + var linked uuid.UUID + if err := db.Pool.QueryRow(ctx, `SELECT attachment_id FROM message_attachments`).Scan(&linked); err != nil || linked != job.AttachmentID { + t.Fatalf("changed link %s %v", linked, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 || failed[0].ErrorCode != "source_missing" { + t.Fatalf("migration failed registry %v %v", failed, err) + } + }) +} diff --git a/internal/platform/apperr/apperr.go b/internal/platform/apperr/apperr.go new file mode 100644 index 00000000..8de0ff3e --- /dev/null +++ b/internal/platform/apperr/apperr.go @@ -0,0 +1,54 @@ +// Package apperr is the application's error vocabulary: every module reports +// failures as a Kind plus a message that is safe to show to clients, and the +// transport layer owns the only mapping from Kind to HTTP status. Causes are +// kept for logs but never rendered. +package apperr + +import "errors" + +type Kind int + +const ( + KindInternal Kind = iota + KindInvalid + KindUnauthorized + KindNotFound + KindConflict + KindPayloadTooLarge + KindUnsupportedMedia + KindUnavailable +) + +type Error struct { + Kind Kind + Message string + Cause error +} + +func (e *Error) Error() string { return e.Message } + +func (e *Error) Unwrap() error { return e.Cause } + +func New(kind Kind, message string) *Error { + return &Error{Kind: kind, Message: message} +} + +func Wrap(kind Kind, message string, cause error) *Error { + return &Error{Kind: kind, Message: message, Cause: cause} +} + +// KindOf classifies any error; foreign errors are internal. +func KindOf(err error) Kind { + if e, ok := errors.AsType[*Error](err); ok { + return e.Kind + } + return KindInternal +} + +func Invalid(message string) *Error { return New(KindInvalid, message) } +func Unauthorized(message string) *Error { return New(KindUnauthorized, message) } +func NotFound(message string) *Error { return New(KindNotFound, message) } +func Conflict(message string) *Error { return New(KindConflict, message) } +func PayloadTooLarge(message string) *Error { return New(KindPayloadTooLarge, message) } +func UnsupportedMedia(message string) *Error { return New(KindUnsupportedMedia, message) } +func Unavailable(message string) *Error { return New(KindUnavailable, message) } diff --git a/internal/platform/apperr/apperr_test.go b/internal/platform/apperr/apperr_test.go new file mode 100644 index 00000000..6de347f7 --- /dev/null +++ b/internal/platform/apperr/apperr_test.go @@ -0,0 +1,49 @@ +package apperr_test + +import ( + "errors" + "fmt" + "testing" + + "messenger/internal/platform/apperr" +) + +func TestKindOfReturnsKindForApplicationError(t *testing.T) { + err := apperr.NotFound("chat not found") + if got := apperr.KindOf(err); got != apperr.KindNotFound { + t.Fatalf("KindOf = %v, want KindNotFound", got) + } + if err.Error() != "chat not found" { + t.Fatalf("Error() = %q", err.Error()) + } +} + +func TestKindOfSeesThroughWrapping(t *testing.T) { + err := fmt.Errorf("service: %w", apperr.Conflict("username taken")) + if got := apperr.KindOf(err); got != apperr.KindConflict { + t.Fatalf("KindOf = %v, want KindConflict", got) + } +} + +func TestKindOfDefaultsToInternal(t *testing.T) { + if got := apperr.KindOf(errors.New("boom")); got != apperr.KindInternal { + t.Fatalf("KindOf = %v, want KindInternal", got) + } + if got := apperr.KindOf(nil); got != apperr.KindInternal { + t.Fatalf("KindOf(nil) = %v, want KindInternal", got) + } +} + +func TestWrapKeepsCause(t *testing.T) { + cause := errors.New("cause") + err := apperr.Wrap(apperr.KindInvalid, "bad input", cause) + if !errors.Is(err, cause) { + t.Fatal("wrapped error must unwrap to cause") + } + if apperr.KindOf(err) != apperr.KindInvalid { + t.Fatal("wrapped error must keep its kind") + } + if err.Error() != "bad input" { + t.Fatalf("Error() = %q, want public message only", err.Error()) + } +} diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go new file mode 100644 index 00000000..2a1e485b --- /dev/null +++ b/internal/platform/config/config.go @@ -0,0 +1,200 @@ +// Package config reads the process configuration from the environment. +// Settings have defaults suited to the lab stand, except credentials, which +// are required and read from files named by *_FILE variables so that their +// values never appear in the environment. An invalid value stops the process +// at start-up with a message naming the variable, never the secret. +package config + +import ( + "fmt" + "log/slog" + "net" + "os" + "strconv" + "strings" + "time" +) + +// Allow a 30s binary request, its independent 3s cleanup, and 5s margin. +const defaultShutdownTimeout = 38 * time.Second + +type Logging struct { + Env string + Level slog.Level +} + +func (l Logging) NewLogger() *slog.Logger { + opts := &slog.HandlerOptions{Level: l.Level} + if l.Env == "dev" { + return slog.New(slog.NewTextHandler(os.Stdout, opts)) + } + return slog.New(slog.NewJSONHandler(os.Stdout, opts)) +} + +type Config struct { + Logging + Host string + Port int + ShutdownTimeout time.Duration + MaxBodyBytes int64 + DatabaseURL string + SessionStoreURL string + SessionTTL time.Duration + // DrainDelay keeps a stopping process serving with readiness off; it must + // exceed the balancer's detection time. + DrainDelay time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DebugAddr string + S3 S3 +} + +type Migrate struct { + Logging + DatabaseURL string +} + +// Lookup reads one variable, reporting whether it was set; os.LookupEnv +// in production, a map in tests. +type Lookup func(key string) (string, bool) + +func Load(lookup Lookup) (Config, error) { + cfg := Config{ + Host: "0.0.0.0", + Port: 8080, + ShutdownTimeout: defaultShutdownTimeout, + MaxBodyBytes: 64 * 1024, + SessionTTL: time.Hour, + DrainDelay: 2 * time.Second, + IdempotencyTTL: 24 * time.Hour, + DBMaxConns: 10, + } + if v, ok := lookup("APP_HOST"); ok { + if strings.TrimSpace(v) == "" { + return Config{}, fmt.Errorf("APP_HOST must not be empty") + } + cfg.Host = v + } + if v, ok := lookup("APP_PORT"); ok { + port, err := strconv.Atoi(v) + if err != nil || port < 1 || port > 65535 { + return Config{}, fmt.Errorf("APP_PORT must be an integer between 1 and 65535, got %q", v) + } + cfg.Port = port + } + var err error + if cfg.Logging, err = loadLogging(lookup); err != nil { + return Config{}, err + } + if v, ok := lookup("APP_SHUTDOWN_TIMEOUT"); ok { + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return Config{}, fmt.Errorf("APP_SHUTDOWN_TIMEOUT must be a positive duration such as 8s, got %q", v) + } + cfg.ShutdownTimeout = d + } + if v, ok := lookup("APP_MAX_BODY_BYTES"); ok { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n <= 0 { + return Config{}, fmt.Errorf("APP_MAX_BODY_BYTES must be a positive integer, got %q", v) + } + cfg.MaxBodyBytes = n + } + if v, ok := lookup("SESSION_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.SessionTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("APP_DRAIN_DELAY"); ok { + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return Config{}, fmt.Errorf("APP_DRAIN_DELAY must be a non-negative duration such as 2s, got %q", v) + } + cfg.DrainDelay = d + } + if v, ok := lookup("IDEMPOTENCY_TTL_SECONDS"); ok { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be a positive whole number of seconds, got %q", v) + } + cfg.IdempotencyTTL = time.Duration(n) * time.Second + } + if v, ok := lookup("DB_MAX_CONNS"); ok { + n, err := strconv.ParseInt(v, 10, 32) + if err != nil || n < 1 || n > 1000 { + return Config{}, fmt.Errorf("DB_MAX_CONNS must be an integer between 1 and 1000, got %q", v) + } + cfg.DBMaxConns = int32(n) + } + if v, ok := lookup("APP_DEBUG_ADDR"); ok && v != "" { + if _, _, err := net.SplitHostPort(v); err != nil { + return Config{}, fmt.Errorf("APP_DEBUG_ADDR must be host:port such as :9090, got %q", v) + } + cfg.DebugAddr = v + } + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { + return Config{}, err + } + if cfg.SessionStoreURL, err = secret(lookup, "SESSION_STORE_URL_FILE"); err != nil { + return Config{}, err + } + if _, ok := lookup("S3_ENDPOINT"); ok { + if cfg.S3, err = LoadS3(lookup); err != nil { + return Config{}, err + } + } + return cfg, nil +} + +func LoadMigrate(lookup Lookup) (Migrate, error) { + var cfg Migrate + var err error + if cfg.Logging, err = loadLogging(lookup); err != nil { + return Migrate{}, err + } + if cfg.DatabaseURL, err = LoadDatabaseURL(lookup); err != nil { + return Migrate{}, err + } + return cfg, nil +} + +func LoadDatabaseURL(lookup Lookup) (string, error) { + return secret(lookup, "DATABASE_URL_FILE") +} + +func loadLogging(lookup Lookup) (Logging, error) { + l := Logging{Env: "lab", Level: slog.LevelInfo} + if v, ok := lookup("APP_ENV"); ok && strings.TrimSpace(v) != "" { + l.Env = v + } + if v, ok := lookup("APP_LOG_LEVEL"); ok { + if err := l.Level.UnmarshalText([]byte(v)); err != nil { + return Logging{}, fmt.Errorf("APP_LOG_LEVEL must be debug, info, warn or error, got %q", v) + } + } + return l, nil +} + +// secret reads the file named by the variable; surrounding whitespace, +// including the trailing newline editors add, is dropped. +func secret(lookup Lookup, key string) (string, error) { + path, ok := lookup(key) + if !ok || strings.TrimSpace(path) == "" { + return "", fmt.Errorf("%s must name a file with the value", key) + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("%s: cannot read %s: %w", key, path, err) + } + value := strings.TrimSpace(string(raw)) + if value == "" { + return "", fmt.Errorf("%s: file %s is empty", key, path) + } + return value, nil +} + +func (c Config) Addr() string { + return net.JoinHostPort(c.Host, strconv.Itoa(c.Port)) +} diff --git a/internal/platform/config/config_test.go b/internal/platform/config/config_test.go new file mode 100644 index 00000000..914795b5 --- /dev/null +++ b/internal/platform/config/config_test.go @@ -0,0 +1,190 @@ +package config_test + +import ( + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/platform/config" +) + +func env(values map[string]string) func(string) (string, bool) { + return func(key string) (string, bool) { + v, ok := values[key] + return v, ok + } +} + +func secretFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +const ( + databaseURL = "postgres://app:s3cret@db:5432/messenger" + sessionStoreURL = "redis://:s3cret@valkey:6379/0" +) + +// required holds the variables without defaults, merged with extra. +func required(t *testing.T, extra map[string]string) map[string]string { + t.Helper() + values := map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "SESSION_STORE_URL_FILE": secretFile(t, sessionStoreURL+"\r\n"), + } + for k, v := range extra { + values[k] = v + } + return values +} + +func TestDefaults(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "0.0.0.0:8080" { + t.Fatalf("Addr = %q", cfg.Addr()) + } + if cfg.Env != "lab" || cfg.Level != slog.LevelInfo { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 38*time.Second || cfg.MaxBodyBytes != 65536 { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.DatabaseURL != databaseURL || cfg.SessionStoreURL != sessionStoreURL { + t.Fatalf("urls = %q, %q; want the file contents without line endings", cfg.DatabaseURL, cfg.SessionStoreURL) + } + if cfg.SessionTTL != time.Hour { + t.Fatalf("SessionTTL = %v, want 1h", cfg.SessionTTL) + } + if cfg.DrainDelay != 2*time.Second || cfg.IdempotencyTTL != 24*time.Hour || cfg.DBMaxConns != 10 || cfg.DebugAddr != "" { + t.Fatalf("lab3 defaults = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } +} + +func TestDefaultShutdownTimeoutCoversBinaryLifecycle(t *testing.T) { + cfg, err := config.Load(env(required(t, nil))) + if err != nil { + t.Fatal(err) + } + // Binary requests can use 30s, followed by 3s independent cleanup. Keep + // another 5s for shutdown scheduling and closing the remaining resources. + budget := 30*time.Second + 3*time.Second + 5*time.Second + if cfg.ShutdownTimeout < budget { + t.Fatalf("default shutdown budget %v cannot cover binary request and cleanup budget %v", cfg.ShutdownTimeout, budget) + } +} + +func TestSecretFilesAreRequiredAndNeverEchoed(t *testing.T) { + for _, key := range []string{"DATABASE_URL_FILE", "SESSION_STORE_URL_FILE"} { + cases := map[string]func(map[string]string){ + "unset": func(v map[string]string) { delete(v, key) }, + "empty path": func(v map[string]string) { v[key] = "" }, + "no file": func(v map[string]string) { v[key] = filepath.Join(t.TempDir(), "missing") }, + "blank file": func(v map[string]string) { v[key] = secretFile(t, " \r\n") }, + } + for name, breakIt := range cases { + values := required(t, nil) + breakIt(values) + _, err := config.Load(env(values)) + if err == nil { + t.Errorf("%s %s: want error", key, name) + continue + } + if !strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "s3cret") { + t.Errorf("%s %s: error must name the variable and hide the secret: %v", key, name, err) + } + } + } +} + +func TestOverrides(t *testing.T) { + cfg, err := config.Load(env(required(t, map[string]string{ + "APP_HOST": "127.0.0.1", + "APP_PORT": "9090", + "APP_ENV": "dev", + "APP_LOG_LEVEL": "debug", + "APP_SHUTDOWN_TIMEOUT": "3s", + "APP_MAX_BODY_BYTES": "1024", + "SESSION_TTL_SECONDS": "300", + "APP_DRAIN_DELAY": "0s", + "IDEMPOTENCY_TTL_SECONDS": "60", + "DB_MAX_CONNS": "25", + "APP_DEBUG_ADDR": ":9090", + }))) + if err != nil { + t.Fatalf("Load error: %v", err) + } + if cfg.Addr() != "127.0.0.1:9090" || cfg.Env != "dev" || cfg.Level != slog.LevelDebug { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.ShutdownTimeout != 3*time.Second || cfg.MaxBodyBytes != 1024 || cfg.SessionTTL != 5*time.Minute { + t.Fatalf("cfg = %+v", cfg) + } + if cfg.DrainDelay != 0 || cfg.IdempotencyTTL != time.Minute || cfg.DBMaxConns != 25 || cfg.DebugAddr != ":9090" { + t.Fatalf("lab3 overrides = drain %v, key ttl %v, pool %d, debug %q", cfg.DrainDelay, cfg.IdempotencyTTL, cfg.DBMaxConns, cfg.DebugAddr) + } +} + +func TestInvalidValuesAreReported(t *testing.T) { + cases := map[string]map[string]string{ + "port text": {"APP_PORT": "abc"}, + "port zero": {"APP_PORT": "0"}, + "port too big": {"APP_PORT": "70000"}, + "level": {"APP_LOG_LEVEL": "loud"}, + "timeout": {"APP_SHUTDOWN_TIMEOUT": "soon"}, + "timeout negative": {"APP_SHUTDOWN_TIMEOUT": "-1s"}, + "body zero": {"APP_MAX_BODY_BYTES": "0"}, + "body text": {"APP_MAX_BODY_BYTES": "big"}, + "empty host": {"APP_HOST": ""}, + "ttl zero": {"SESSION_TTL_SECONDS": "0"}, + "ttl negative": {"SESSION_TTL_SECONDS": "-5"}, + "ttl text": {"SESSION_TTL_SECONDS": "hour"}, + "ttl fraction": {"SESSION_TTL_SECONDS": "1.5"}, + "drain negative": {"APP_DRAIN_DELAY": "-1s"}, + "drain text": {"APP_DRAIN_DELAY": "abc"}, + "key ttl zero": {"IDEMPOTENCY_TTL_SECONDS": "0"}, + "key ttl text": {"IDEMPOTENCY_TTL_SECONDS": "day"}, + "pool zero": {"DB_MAX_CONNS": "0"}, + "pool too big": {"DB_MAX_CONNS": "1001"}, + "pool text": {"DB_MAX_CONNS": "many"}, + "debug addr": {"APP_DEBUG_ADDR": "nonsense"}, + } + for name, values := range cases { + _, err := config.Load(env(required(t, values))) + if err == nil { + t.Errorf("%s: want error", name) + continue + } + for key := range values { + if !strings.Contains(err.Error(), key) { + t.Errorf("%s: error %q does not name %s", name, err, key) + } + } + } +} + +func TestLoadMigrateNeedsOnlyTheDatabaseURL(t *testing.T) { + cfg, err := config.LoadMigrate(env(map[string]string{ + "DATABASE_URL_FILE": secretFile(t, databaseURL+"\n"), + "APP_ENV": "dev", + "APP_LOG_LEVEL": "warn", + })) + if err != nil { + t.Fatalf("LoadMigrate error: %v", err) + } + if cfg.DatabaseURL != databaseURL || cfg.Env != "dev" || cfg.Level != slog.LevelWarn { + t.Fatalf("cfg = %+v", cfg) + } + if _, err := config.LoadMigrate(env(map[string]string{})); err == nil || !strings.Contains(err.Error(), "DATABASE_URL_FILE") { + t.Fatalf("missing database url: err = %v", err) + } +} diff --git a/internal/platform/config/jobs.go b/internal/platform/config/jobs.go new file mode 100644 index 00000000..1e550e49 --- /dev/null +++ b/internal/platform/config/jobs.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "strconv" + "time" +) + +type Jobs struct { + URL string + Concurrency, MaxAttempts int + Timeout time.Duration +} + +func LoadBrokerURL(lookup Lookup) (string, error) { + return secret(lookup, "BROKER_URL_FILE") +} + +func LoadJobs(lookup Lookup) (Jobs, error) { + url, err := LoadBrokerURL(lookup) + if err != nil { + return Jobs{}, err + } + c := Jobs{URL: url, Concurrency: 2, MaxAttempts: 5} + seconds := 60 + for _, p := range []struct { + key string + value *int + max int + }{ + {"WORKER_CONCURRENCY", &c.Concurrency, 2}, + {"JOB_MAX_ATTEMPTS", &c.MaxAttempts, 100}, + {"JOB_TIMEOUT_SECONDS", &seconds, 3600}, + } { + if s, ok := lookup(p.key); ok { + n, err := strconv.Atoi(s) + if err != nil || n < 1 || n > p.max { + return Jobs{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max) + } + *p.value = n + } + } + c.Timeout = time.Duration(seconds) * time.Second + return c, nil +} diff --git a/internal/platform/config/jobs_test.go b/internal/platform/config/jobs_test.go new file mode 100644 index 00000000..0f5d2a82 --- /dev/null +++ b/internal/platform/config/jobs_test.go @@ -0,0 +1,28 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestLoadJobs(t *testing.T) { + path := filepath.Join(t.TempDir(), "url") + if err := os.WriteFile(path, []byte("amqp://worker:secret@broker:5672/\n"), 0600); err != nil { + t.Fatal(err) + } + values := map[string]string{"BROKER_URL_FILE": path} + lookup := func(k string) (string, bool) { v, ok := values[k]; return v, ok } + cfg, err := LoadJobs(lookup) + if err != nil || cfg.Timeout != 60*time.Second || cfg.MaxAttempts != 5 || cfg.Concurrency < 1 || cfg.URL != "amqp://worker:secret@broker:5672/" { + t.Fatalf("unexpected defaults or secret read, err=%v", err) + } + for _, key := range []string{"JOB_TIMEOUT_SECONDS", "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS"} { + values[key] = "0" + if _, err := LoadJobs(lookup); err == nil { + t.Errorf("%s accepted zero", key) + } + delete(values, key) + } +} diff --git a/internal/platform/config/s3.go b/internal/platform/config/s3.go new file mode 100644 index 00000000..8a4d9a22 --- /dev/null +++ b/internal/platform/config/s3.go @@ -0,0 +1,71 @@ +package config + +import ( + "fmt" + "math" + "strconv" + "time" + + "messenger/internal/imaging" +) + +type S3 struct { + Endpoint, Region, Bucket string + AccessKey, SecretKey string + MaxBytes, MaxPixels int64 + StagingRetention time.Duration +} + +func LoadS3(lookup Lookup) (S3, error) { + cfg := S3{ + Endpoint: "http://s3:8333", Region: "us-east-1", Bucket: "messenger-attachments", + MaxBytes: imaging.MaxUploadBytes, MaxPixels: imaging.MaxImagePixels, StagingRetention: 24 * time.Hour, + } + for key, dest := range map[string]*string{"S3_ENDPOINT": &cfg.Endpoint, "S3_REGION": &cfg.Region, "S3_BUCKET": &cfg.Bucket} { + if v, ok := lookup(key); ok { + *dest = v + } + } + var err error + if cfg.AccessKey, err = secret(lookup, "S3_ACCESS_KEY_ID_FILE"); err != nil { + return S3{}, err + } + if cfg.SecretKey, err = secret(lookup, "S3_SECRET_ACCESS_KEY_FILE"); err != nil { + return S3{}, err + } + for _, p := range []struct { + key string + dest *int64 + max int64 + }{{"MAX_UPLOAD_BYTES", &cfg.MaxBytes, imaging.MaxUploadBytes}, {"MAX_IMAGE_PIXELS", &cfg.MaxPixels, imaging.MaxImagePixels}} { + if v, ok := lookup(p.key); ok { + n, e := strconv.ParseInt(v, 10, 64) + if e != nil || n < 1 || n > p.max { + return S3{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max) + } + *p.dest = n + } + } + if raw, ok := lookup("STAGING_RETENTION_SECONDS"); ok { + const maxSeconds = math.MaxInt64 / int64(time.Second) + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil || n < 1 || n > maxSeconds { + return S3{}, fmt.Errorf("STAGING_RETENTION_SECONDS must be between 1 and %d", maxSeconds) + } + cfg.StagingRetention = time.Duration(n) * time.Second + } + return cfg, nil +} + +// ReconcileS3 selects the separate listing identity used by background scans. +func ReconcileS3(lookup Lookup) Lookup { + return func(key string) (string, bool) { + if key == "S3_ACCESS_KEY_ID_FILE" { + return lookup("S3_RECONCILE_ACCESS_KEY_ID_FILE") + } + if key == "S3_SECRET_ACCESS_KEY_FILE" { + return lookup("S3_RECONCILE_SECRET_ACCESS_KEY_FILE") + } + return lookup(key) + } +} diff --git a/internal/platform/config/s3_test.go b/internal/platform/config/s3_test.go new file mode 100644 index 00000000..72055ef5 --- /dev/null +++ b/internal/platform/config/s3_test.go @@ -0,0 +1,46 @@ +package config_test + +import ( + "messenger/internal/platform/config" + "testing" + "time" +) + +func TestS3LimitsAndSecretFiles(t *testing.T) { + values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")} + cfg, err := config.LoadS3(env(values)) + if err != nil { + t.Fatal(err) + } + if cfg.AccessKey != "access" || cfg.SecretKey != "private" || cfg.MaxBytes != 10485760 || cfg.MaxPixels != 40000000 { + t.Fatal("S3 defaults or secret trimming broken") + } + for _, tc := range []struct{ k, v string }{{"MAX_UPLOAD_BYTES", "0"}, {"MAX_UPLOAD_BYTES", "10485761"}, {"MAX_IMAGE_PIXELS", "40000001"}, {"MAX_IMAGE_PIXELS", "x"}} { + v := map[string]string{} + for k, x := range values { + v[k] = x + } + v[tc.k] = tc.v + if _, err := config.LoadS3(env(v)); err == nil { + t.Fatalf("accepted %s=%s", tc.k, tc.v) + } + } +} +func TestS3RejectsInvalidStagingRetention(t *testing.T) { + values := map[string]string{"S3_ACCESS_KEY_ID_FILE": secretFile(t, "access\n"), "S3_SECRET_ACCESS_KEY_FILE": secretFile(t, "private\n")} + cfg, err := config.LoadS3(env(values)) + if err != nil || cfg.StagingRetention != 24*time.Hour { + t.Fatalf("default retention %v %v", cfg.StagingRetention, err) + } + values["STAGING_RETENTION_SECONDS"] = "7200" + cfg, err = config.LoadS3(env(values)) + if err != nil || cfg.StagingRetention != 2*time.Hour { + t.Fatalf("custom retention %v %v", cfg.StagingRetention, err) + } + for _, value := range []string{"0", "-1", "abc", "9223372036854775807"} { + values["STAGING_RETENTION_SECONDS"] = value + if _, err := config.LoadS3(env(values)); err == nil { + t.Errorf("accepted invalid retention %s", value) + } + } +} diff --git a/internal/platform/faultbarrier/barrier.go b/internal/platform/faultbarrier/barrier.go new file mode 100644 index 00000000..ee9ee120 --- /dev/null +++ b/internal/platform/faultbarrier/barrier.go @@ -0,0 +1,38 @@ +// Package faultbarrier provides one-shot synchronization for controlled process-failure experiments. +package faultbarrier + +import ( + "context" + "os" + "time" +) + +// Wait claims an armed phase and blocks until its hit file disappears. +// A nonempty id identifies a newly staged attachment before its job exists. +func Wait(ctx context.Context, base, id string) error { + if err := os.Rename(base+".arm", base+".hit"); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if id != "" { + if err := os.WriteFile(base+".id", []byte(id), 0600); err != nil { + return err + } + } + t := time.NewTicker(50 * time.Millisecond) + defer t.Stop() + for { + if _, err := os.Stat(base + ".hit"); os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } +} diff --git a/internal/platform/faultbarrier/barrier_test.go b/internal/platform/faultbarrier/barrier_test.go new file mode 100644 index 00000000..9c78a9d5 --- /dev/null +++ b/internal/platform/faultbarrier/barrier_test.go @@ -0,0 +1,48 @@ +package faultbarrier + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +func TestArmedBarrierRecordsIDAndWaitsForRelease(t *testing.T) { + base := filepath.Join(t.TempDir(), "digest-staged") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := Wait(ctx, base, "unused"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(base + ".id"); !os.IsNotExist(err) { + t.Fatal("unarmed barrier created identity") + } + if err := os.WriteFile(base+".arm", nil, 0600); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- Wait(ctx, base, "attachment-id") }() + for { + b, err := os.ReadFile(base + ".id") + if err == nil && string(b) == "attachment-id" { + break + } + select { + case <-ctx.Done(): + t.Fatal("missing staged identity") + case <-time.After(10 * time.Millisecond): + } + } + select { + case <-done: + t.Fatal("continued before release") + default: + } + if err := os.Remove(base + ".hit"); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/platform/httpx/decode.go b/internal/platform/httpx/decode.go new file mode 100644 index 00000000..ce8eeae7 --- /dev/null +++ b/internal/platform/httpx/decode.go @@ -0,0 +1,70 @@ +package httpx + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strings" + "unicode/utf8" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +// DecodeJSON reads exactly one JSON object into dst and enforces what gin's +// binding does not: the media type must be application/json (415), invalid +// UTF-8, unknown fields, wrong types, malformed syntax and trailing data are +// 400, an oversized body is 413, and messages never expose Go type names. +func DecodeJSON(c *gin.Context, dst any) error { + mediaType, _, err := mime.ParseMediaType(c.GetHeader("Content-Type")) + if err != nil || mediaType != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + // The body is already capped by the bodyLimit middleware, so reading it + // whole is bounded. + body, err := io.ReadAll(c.Request.Body) + if err != nil { + return decodeError(err) + } + // encoding/json silently replaces invalid UTF-8 with U+FFFD; the contract + // and RFC 8259 require UTF-8, so such a body is rejected, not repaired. + if !utf8.Valid(body) { + return apperr.Invalid("request body must be valid UTF-8") + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return decodeError(err) + } + if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return apperr.Invalid("unexpected data after JSON body") + } + return nil +} + +func decodeError(err error) error { + if tooLarge, ok := errors.AsType[*http.MaxBytesError](err); ok { + return apperr.Wrap(apperr.KindPayloadTooLarge, fmt.Sprintf("request body must be at most %d bytes", tooLarge.Limit), err) + } + if errors.Is(err, io.EOF) { + return apperr.Wrap(apperr.KindInvalid, "request body is required", err) + } + if wrongType, ok := errors.AsType[*json.UnmarshalTypeError](err); ok { + if wrongType.Field == "" { + return apperr.Wrap(apperr.KindInvalid, "request body must be a JSON object", err) + } + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("field %q has wrong type", wrongType.Field), err) + } + // encoding/json reports unknown fields as a plain error without a type. + const unknownPrefix = "json: unknown field " + if msg := err.Error(); strings.HasPrefix(msg, unknownPrefix) { + field := strings.Trim(strings.TrimPrefix(msg, unknownPrefix), `"`) + return apperr.Wrap(apperr.KindInvalid, fmt.Sprintf("unknown field %q", field), err) + } + return apperr.Wrap(apperr.KindInvalid, "malformed JSON body", err) +} diff --git a/internal/platform/httpx/engine.go b/internal/platform/httpx/engine.go new file mode 100644 index 00000000..4f1f89ad --- /dev/null +++ b/internal/platform/httpx/engine.go @@ -0,0 +1,50 @@ +// Package httpx is the transport layer over gin: request ids, access +// logging, panic recovery, body limits, strict JSON decoding and the single +// error format of the API contract. gin is used for routing and the +// middleware chain; everything here exists because the contract demands a +// specific behaviour that gin's own helpers do not provide. +package httpx + +import ( + "log/slog" + "net/http" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +const ( + keyRequestID = "messenger.request_id" + keyLogger = "messenger.logger" +) + +// NewEngine builds an engine whose every response, including routing +// failures and panics, carries X-Instance-Id, X-Request-Id and the contract +// error body. +func NewEngine(log *slog.Logger, maxBodyBytes int64, instanceID string) *gin.Engine { + gin.SetMode(gin.ReleaseMode) + e := gin.New() + // Redirects would answer with text/plain and without a request id. + e.RedirectTrailingSlash = false + e.RedirectFixedPath = false + e.HandleMethodNotAllowed = true + + e.Use(instance(instanceID), requestID(log), accessLog(), recovery(), bodyLimit(maxBodyBytes)) + e.NoRoute(func(c *gin.Context) { Error(c, apperr.NotFound("route not found")) }) + e.NoMethod(func(c *gin.Context) { Fail(c, http.StatusMethodNotAllowed, "invalid_request", "method not allowed") }) + return e +} + +func RequestID(c *gin.Context) string { + return c.GetString(keyRequestID) +} + +func logger(c *gin.Context) *slog.Logger { + if l, ok := c.Get(keyLogger); ok { + if log, ok := l.(*slog.Logger); ok { + return log + } + } + return slog.Default() +} diff --git a/internal/platform/httpx/httpx_test.go b/internal/platform/httpx/httpx_test.go new file mode 100644 index 00000000..30c64aca --- /dev/null +++ b/internal/platform/httpx/httpx_test.go @@ -0,0 +1,409 @@ +package httpx_test + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/httpx" +) + +type errorBody struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type payload struct { + Text string `json:"text"` + Count int `json:"count"` +} + +const testInstance = "0123456789abcdef" + +func newEngine(t *testing.T, logs *bytes.Buffer) *gin.Engine { + t.Helper() + return newEngineAt(t, logs, slog.LevelDebug) +} + +func newEngineAt(t *testing.T, logs *bytes.Buffer, level slog.Level) *gin.Engine { + t.Helper() + if logs == nil { + logs = &bytes.Buffer{} + } + log := slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: level})) + e := httpx.NewEngine(log, 64, testInstance) + e.GET("/ok", func(c *gin.Context) { httpx.JSON(c, http.StatusOK, gin.H{"text": ""}) }) + e.GET("/panic", func(c *gin.Context) { panic("secret detail") }) + e.GET("/fail", func(c *gin.Context) { httpx.Error(c, errors.New("db password leaked")) }) + e.GET("/notfound", func(c *gin.Context) { httpx.Error(c, apperr.NotFound("chat not found")) }) + e.GET("/who", func(c *gin.Context) { + httpx.SetUserID(c, uuid.MustParse("11111111-1111-4111-8111-111111111111")) + id, ok := httpx.UserID(c) + httpx.JSON(c, http.StatusOK, gin.H{"id": id.String(), "ok": ok}) + }) + e.POST("/decode", func(c *gin.Context) { + var p payload + if err := httpx.DecodeJSON(c, &p); err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, p) + }) + return e +} + +func do(t *testing.T, e *gin.Engine, method, path string, body string, headers map[string]string) (*httptest.ResponseRecorder, errorBody) { + t.Helper() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, path, reader) + for k, v := range headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + var eb errorBody + if rec.Code >= 400 { + if err := json.Unmarshal(rec.Body.Bytes(), &eb); err != nil { + t.Fatalf("%s %s: error body is not JSON: %v: %s", method, path, err, rec.Body.String()) + } + } + return rec, eb +} + +func assertError(t *testing.T, rec *httptest.ResponseRecorder, eb errorBody, status int, code string) { + t.Helper() + if rec.Code != status { + t.Fatalf("status = %d, want %d (body %s)", rec.Code, status, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Fatalf("Content-Type = %q", ct) + } + rid := rec.Header().Get("X-Request-Id") + if rid == "" { + t.Fatal("missing X-Request-Id header") + } + if eb.Error.RequestID != rid { + t.Fatalf("body request_id %q != header %q", eb.Error.RequestID, rid) + } + if eb.Error.Code != code || eb.Error.Message == "" { + t.Fatalf("error = %+v, want code %q with a message", eb.Error, code) + } +} + +// The balancer spreads one client's requests over the replicas; the header +// is how a client tells which process answered, so no response may lack it. +func TestEveryResponseCarriesInstanceID(t *testing.T) { + e := newEngine(t, nil) + for _, r := range []struct{ method, path string }{ + {http.MethodGet, "/ok"}, + {http.MethodGet, "/missing"}, + {http.MethodDelete, "/ok"}, + {http.MethodGet, "/panic"}, + {http.MethodGet, "/fail"}, + {http.MethodPost, "/decode"}, + } { + rec, _ := do(t, e, r.method, r.path, "", nil) + if got := rec.Header().Get("X-Instance-Id"); got != testInstance { + t.Errorf("%s %s (status %d): X-Instance-Id = %q, want %q", r.method, r.path, rec.Code, got, testInstance) + } + } +} + +func TestUnknownRouteIsJSON404(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/nope", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestTrailingSlashIsNotRedirected(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/ok/", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") +} + +func TestWrongMethodIsJSON405WithAllow(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodDelete, "/ok", "", nil) + assertError(t, rec, eb, http.StatusMethodNotAllowed, "invalid_request") + if !strings.Contains(rec.Header().Get("Allow"), "GET") { + t.Fatalf("Allow = %q, want GET", rec.Header().Get("Allow")) + } +} + +func TestPanicBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/panic", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "secret detail") { + t.Fatal("panic value leaked to the client") + } + if !strings.Contains(logs.String(), "secret detail") { + t.Fatal("panic must be logged for operators") + } +} + +func TestForeignErrorBecomesJSON500WithoutDetails(t *testing.T) { + logs := &bytes.Buffer{} + rec, eb := do(t, newEngine(t, logs), http.MethodGet, "/fail", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") + if strings.Contains(rec.Body.String(), "password") { + t.Fatal("internal error text leaked to the client") + } + if !strings.Contains(logs.String(), "db password leaked") { + t.Fatal("cause must be logged for operators") + } +} + +func TestApplicationErrorMapping(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodGet, "/notfound", "", nil) + assertError(t, rec, eb, http.StatusNotFound, "not_found") + if eb.Error.Message != "chat not found" { + t.Fatalf("message = %q", eb.Error.Message) + } +} + +func TestSuccessResponseHeadersAndEscaping(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json; charset=utf-8" { + t.Fatalf("Content-Type = %q", ct) + } + if rec.Header().Get("X-Request-Id") == "" { + t.Fatal("missing X-Request-Id on success") + } + if !strings.Contains(rec.Body.String(), `""`) { + t.Fatalf("HTML escaping must be off, body %s", rec.Body.String()) + } +} + +// The balancer sets X-Request-Id on every request, overwriting what the +// client sent; the API keeps it so that one id spans both logs. +func TestRequestIDFromTheBalancerIsKept(t *testing.T) { + const fromLB = "0123456789abcdef0123456789abcdef" + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": fromLB}) + if got := rec.Header().Get("X-Request-Id"); got != fromLB { + t.Fatalf("X-Request-Id = %q, want %q", got, fromLB) + } +} + +func TestMalformedRequestIDIsReplaced(t *testing.T) { + for _, bad := range []string{"has space", "semi;colon", "tab\there", strings.Repeat("a", 65)} { + rec, _ := do(t, newEngine(t, nil), http.MethodGet, "/ok", "", map[string]string{"X-Request-Id": bad}) + if got := rec.Header().Get("X-Request-Id"); got == "" || got == bad { + t.Errorf("X-Request-Id for %q = %q, want a generated value", bad, got) + } + } +} + +func TestDecodeJSONHappyPath(t *testing.T) { + rec, _ := do(t, newEngine(t, nil), http.MethodPost, "/decode", `{"text":"Привет","count":2}`, map[string]string{"Content-Type": "application/json; charset=utf-8"}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"Привет"`) { + t.Fatalf("status %d body %s", rec.Code, rec.Body.String()) + } +} + +func TestDecodeJSONFailures(t *testing.T) { + jsonCT := map[string]string{"Content-Type": "application/json"} + cases := []struct { + name string + body string + headers map[string]string + status int + code string + contain string + }{ + {"no content type", `{"text":"x"}`, nil, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"wrong content type", `{"text":"x"}`, map[string]string{"Content-Type": "text/plain"}, http.StatusUnsupportedMediaType, "unsupported_media_type", ""}, + {"empty body", "", jsonCT, http.StatusBadRequest, "invalid_request", "body"}, + {"malformed", `{"text":`, jsonCT, http.StatusBadRequest, "invalid_request", "malformed"}, + {"invalid utf-8", "{\"text\":\"bad \xff byte\"}", jsonCT, http.StatusBadRequest, "invalid_request", "UTF-8"}, + {"unknown field", `{"text":"x","extra":1}`, jsonCT, http.StatusBadRequest, "invalid_request", "extra"}, + {"wrong type", `{"text":123}`, jsonCT, http.StatusBadRequest, "invalid_request", "text"}, + {"trailing data", `{"text":"x"} {"text":"y"}`, jsonCT, http.StatusBadRequest, "invalid_request", "after"}, + {"array instead of object", `[1,2]`, jsonCT, http.StatusBadRequest, "invalid_request", ""}, + {"too large", `{"text":"` + strings.Repeat("x", 100) + `"}`, jsonCT, http.StatusRequestEntityTooLarge, "payload_too_large", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec, eb := do(t, newEngine(t, nil), http.MethodPost, "/decode", tc.body, tc.headers) + assertError(t, rec, eb, tc.status, tc.code) + if tc.contain != "" && !strings.Contains(eb.Error.Message, tc.contain) { + t.Fatalf("message %q must mention %q", eb.Error.Message, tc.contain) + } + if strings.Contains(eb.Error.Message, "httpx_test") || strings.Contains(eb.Error.Message, "payload") { + t.Fatalf("message must not leak Go type names: %q", eb.Error.Message) + } + }) + } +} + +func TestAccessLogHasRequestFieldsButNoBody(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + rec, _ := do(t, e, http.MethodPost, "/decode", `{"text":"confidential"}`, map[string]string{"Content-Type": "application/json"}) + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + for _, key := range []string{"request_id", "method", "path", "status", "duration_ms"} { + if _, ok := entry[key]; !ok { + t.Errorf("log entry missing %q: %s", key, line) + } + } + if entry["request_id"] != rec.Header().Get("X-Request-Id") { + t.Errorf("log request_id %v != header %q", entry["request_id"], rec.Header().Get("X-Request-Id")) + } + if strings.Contains(logs.String(), "confidential") { + t.Fatal("request body must never be logged") + } +} + +func TestAccessLogLevelFollowsStatus(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngineAt(t, logs, slog.LevelInfo) + + do(t, e, http.MethodGet, "/ok", "", nil) + if strings.Contains(logs.String(), `"msg":"request"`) { + t.Fatalf("successful requests must be logged at debug level only, got %s", logs.String()) + } + do(t, e, http.MethodGet, "/notfound", "", nil) + if entry := logEntry(t, logs); entry["level"] != "INFO" || entry["status"] != float64(404) { + t.Fatalf("client error must be logged at info level with its status, got %v", entry) + } + do(t, e, http.MethodGet, "/fail", "", nil) + if entry := logEntry(t, logs); entry["level"] != "ERROR" || entry["status"] != float64(500) { + t.Fatalf("server error must be logged at error level with its status, got %v", entry) + } + + verbose := &bytes.Buffer{} + do(t, newEngineAt(t, verbose, slog.LevelDebug), http.MethodGet, "/ok", "", nil) + if entry := logEntry(t, verbose); entry["level"] != "DEBUG" || entry["status"] != float64(200) { + t.Fatalf("at debug level successful requests must be logged, got %v", entry) + } +} + +func TestUserIDRoundTripAndAccessLogAttribution(t *testing.T) { + logs := &bytes.Buffer{} + rec, _ := do(t, newEngine(t, logs), http.MethodGet, "/who", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"ok":true`) { + t.Fatalf("UserID must return what SetUserID stored: %d %s", rec.Code, rec.Body.String()) + } + if entry := logEntry(t, logs); entry["user_id"] != "11111111-1111-4111-8111-111111111111" { + t.Fatalf("access log must attribute the request to the user, got %v", entry) + } + anonymous := &bytes.Buffer{} + do(t, newEngine(t, anonymous), http.MethodGet, "/ok", "", nil) + if _, present := logEntry(t, anonymous)["user_id"]; present { + t.Fatal("anonymous requests must not carry user_id") + } +} + +func TestWithUserPassesTheIdentifiedCaller(t *testing.T) { + actor := uuid.MustParse("22222222-2222-4222-8222-222222222222") + e := newEngine(t, nil) + e.GET("/actor", + func(c *gin.Context) { httpx.SetUserID(c, actor) }, + httpx.WithUser(func(c *gin.Context, got uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{"id": got.String()}) + })) + rec, _ := do(t, e, http.MethodGet, "/actor", "", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), actor.String()) { + t.Fatalf("handler did not receive the caller: %d %s", rec.Code, rec.Body.String()) + } +} + +// Reaching WithUser without an authentication middleware in front is a +// wiring mistake, so it must not be blamed on the client with a 401. +func TestWithUserWithoutIdentityIsAWiringError(t *testing.T) { + e := newEngine(t, nil) + e.GET("/actor", httpx.WithUser(func(c *gin.Context, _ uuid.UUID) { + httpx.JSON(c, http.StatusOK, gin.H{}) + })) + rec, eb := do(t, e, http.MethodGet, "/actor", "", nil) + assertError(t, rec, eb, http.StatusInternalServerError, "internal_error") +} + +// A 5xx tells the client only the safe message; the operator must still see +// why the dependency failed, so the cause goes to the log. +func TestUnavailableCauseIsLoggedButNotReturned(t *testing.T) { + logs := &bytes.Buffer{} + e := newEngine(t, logs) + e.GET("/down", func(c *gin.Context) { + cause := errors.New("dial tcp 10.0.0.7:5432: connect: connection refused") + httpx.Error(c, fmt.Errorf("select user: %w", apperr.Wrap(apperr.KindUnavailable, "database unavailable", cause))) + }) + rec, eb := do(t, e, http.MethodGet, "/down", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + // Neither the cause nor the wrapping context of internal operations + // ("select user: ...") may reach the client. + if eb.Error.Message != "database unavailable" { + t.Fatalf("client message = %q, want only the application message", eb.Error.Message) + } + if !strings.Contains(logs.String(), "connection refused") || !strings.Contains(logs.String(), eb.Error.RequestID) { + t.Fatalf("log must carry the cause and the request id, got:\n%s", logs.String()) + } +} + +// Everything below Timeout sees one deadline, so waiting on a dependency +// ends with the request instead of holding the client. +func TestTimeoutBoundsTheRequestContext(t *testing.T) { + e := newEngine(t, nil) + e.GET("/slow", httpx.Timeout(50*time.Millisecond), func(c *gin.Context) { + select { + case <-c.Request.Context().Done(): + httpx.Error(c, apperr.Wrap(apperr.KindUnavailable, "gave up", c.Request.Context().Err())) + case <-time.After(5 * time.Second): + httpx.JSON(c, http.StatusOK, gin.H{}) + } + }) + start := time.Now() + rec, eb := do(t, e, http.MethodGet, "/slow", "", nil) + assertError(t, rec, eb, http.StatusServiceUnavailable, "unavailable") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("request took %v, want it cut at the 50ms deadline", elapsed) + } +} + +func logEntry(t *testing.T, logs *bytes.Buffer) map[string]any { + t.Helper() + var entry map[string]any + line := lastLine(logs.String()) + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("log line is not JSON: %v: %s", err, line) + } + return entry +} + +func TestTimestampFormat(t *testing.T) { + ts := time.Date(2026, 9, 1, 12, 0, 0, 0, time.FixedZone("MSK", 3*3600)) + if got := httpx.Timestamp(ts); got != "2026-09-01T09:00:00.000Z" { + t.Fatalf("Timestamp = %q", got) + } + ts = time.Date(2026, 9, 1, 12, 0, 0, 7_000_000, time.UTC) + if got := httpx.Timestamp(ts); got != "2026-09-01T12:00:00.007Z" { + t.Fatalf("Timestamp = %q", got) + } +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} diff --git a/internal/platform/httpx/identity.go b/internal/platform/httpx/identity.go new file mode 100644 index 00000000..a90fd50c --- /dev/null +++ b/internal/platform/httpx/identity.go @@ -0,0 +1,42 @@ +package httpx + +import ( + "errors" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +const keyUserID = "messenger.user_id" + +// SetUserID records the authenticated caller for the handlers behind the +// authentication middleware and for the access log. +func SetUserID(c *gin.Context, id uuid.UUID) { + c.Set(keyUserID, id) +} + +func UserID(c *gin.Context) (uuid.UUID, bool) { + v, ok := c.Get(keyUserID) + if !ok { + return uuid.Nil, false + } + id, ok := v.(uuid.UUID) + return id, ok +} + +// UserHandler is a handler that needs the authenticated caller. +type UserHandler func(c *gin.Context, actor uuid.UUID) + +// WithUser adapts a UserHandler to gin. Reaching it without the +// authentication middleware in front is a wiring mistake, so it is reported +// as an internal error rather than blamed on the client with a 401. +func WithUser(h UserHandler) gin.HandlerFunc { + return func(c *gin.Context) { + actor, ok := UserID(c) + if !ok { + Error(c, errors.New("route is missing the authentication middleware")) + return + } + h(c, actor) + } +} diff --git a/internal/platform/httpx/middleware.go b/internal/platform/httpx/middleware.go new file mode 100644 index 00000000..052e0d2d --- /dev/null +++ b/internal/platform/httpx/middleware.go @@ -0,0 +1,136 @@ +package httpx + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "net/http" + "regexp" + "runtime/debug" + "time" + + "github.com/gin-gonic/gin" +) + +// instance names the process that answered, so a client behind the load +// balancer can tell the replicas apart. +func instance(id string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("X-Instance-Id", id) + c.Next() + } +} + +var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// requestID keeps the X-Request-Id set by the load balancer, which overwrites +// any value a client sends; a malformed value is replaced, so a direct caller +// cannot inject into the logs. +func requestID(log *slog.Logger) gin.HandlerFunc { + return func(c *gin.Context) { + id := c.GetHeader("X-Request-Id") + if !requestIDPattern.MatchString(id) { + id = newRequestID() + } + c.Set(keyRequestID, id) + c.Set(keyLogger, log) + c.Writer.Header().Set("X-Request-Id", id) + c.Next() + } +} + +func newRequestID() string { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + panic("httpx: random source unavailable: " + err.Error()) + } + return hex.EncodeToString(raw[:]) +} + +// accessLog writes one line per request at a level that follows the +// outcome (2xx debug, 4xx info, 5xx error), so the ordinary log level decides +// how much traffic is recorded. Headers, query strings and bodies are never +// recorded. time.Since uses the monotonic clock, so wall clock adjustments +// cannot distort the duration. +func accessLog() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + + status := c.Writer.Status() + level := slog.LevelDebug + switch { + case status >= http.StatusInternalServerError: + level = slog.LevelError + case status >= http.StatusBadRequest: + level = slog.LevelInfo + } + log := logger(c) + ctx := c.Request.Context() + if !log.Enabled(ctx, level) { + return + } + attrs := []any{ + "request_id", RequestID(c), + "method", c.Request.Method, + "path", c.Request.URL.Path, + "status", status, + "duration_ms", float64(time.Since(start).Microseconds()) / 1000, + "bytes", c.Writer.Size(), + } + if id, ok := UserID(c); ok { + attrs = append(attrs, "user_id", id.String()) + } + log.Log(ctx, level, "request", attrs...) + } +} + +// recovery turns a panic into a contract 500; the stack goes to the log only. +func recovery() gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + if r := recover(); r != nil { + logger(c).Error("panic recovered", + "request_id", RequestID(c), + "panic", fmt.Sprint(r), + "stack", string(debug.Stack()), + ) + if !c.Writer.Written() { + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + } + c.Abort() + } + }() + c.Next() + } +} + +// Timeout bounds the whole request with one deadline: waiting for a pooled +// connection, SQL statements and session lookups all observe it, so a hung +// dependency ends in a 503 within d instead of holding the client. +func Timeout(d time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), d) + defer cancel() + c.Request = c.Request.WithContext(ctx) + c.Next() + } +} + +// bodyLimit caps request bodies; exceeding it surfaces from the JSON +// decoder as *http.MaxBytesError. +func bodyLimit(maxBytes int64) gin.HandlerFunc { + return func(c *gin.Context) { + // The authenticated binary handler installs its own 10 MiB limit. + if c.Request.Method == http.MethodPost && c.FullPath() == "/api/v1/chats/:chat_id/attachments" { + c.Next() + return + } + if c.Request.Body != nil { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) + } + c.Next() + } +} diff --git a/internal/platform/httpx/respond.go b/internal/platform/httpx/respond.go new file mode 100644 index 00000000..30f49354 --- /dev/null +++ b/internal/platform/httpx/respond.go @@ -0,0 +1,87 @@ +package httpx + +import ( + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/platform/apperr" +) + +type errorEnvelope struct { + Error errorDetail `json:"error"` +} + +type errorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type statusCode struct { + status int + code string +} + +var kinds = map[apperr.Kind]statusCode{ + apperr.KindInvalid: {http.StatusBadRequest, "invalid_request"}, + apperr.KindUnauthorized: {http.StatusUnauthorized, "unauthorized"}, + apperr.KindNotFound: {http.StatusNotFound, "not_found"}, + apperr.KindConflict: {http.StatusConflict, "conflict"}, + apperr.KindPayloadTooLarge: {http.StatusRequestEntityTooLarge, "payload_too_large"}, + apperr.KindUnsupportedMedia: {http.StatusUnsupportedMediaType, "unsupported_media_type"}, + apperr.KindUnavailable: {http.StatusServiceUnavailable, "unavailable"}, +} + +// JSON is the single place that fixes the response media type; PureJSON +// keeps non-ASCII and HTML characters verbatim. +func JSON(c *gin.Context, status int, body any) { + c.PureJSON(status, body) +} + +// Fail writes the contract error body and stops the handler chain. +func Fail(c *gin.Context, status int, code, message string) { + JSON(c, status, errorEnvelope{Error: errorDetail{ + Code: code, + Message: message, + RequestID: RequestID(c), + }}) + c.Abort() +} + +// Error maps an application error to its status. Anything that is not an +// application error is logged with its text and reported as an opaque 500, +// so internal details never reach the client. An application error shows the +// client only its own message, never the context it was wrapped in on the +// way up; when it is answered with a 5xx (a dependency outage) the full +// chain and the cause go to the log for the operator. +func Error(c *gin.Context, err error) { + appErr, isApp := errors.AsType[*apperr.Error](err) + var mapped statusCode + known := false + if isApp { + mapped, known = kinds[appErr.Kind] + } + if !known { + logger(c).Error("request failed", "request_id", RequestID(c), "error", err.Error()) + Fail(c, http.StatusInternalServerError, "internal_error", "internal error") + return + } + if mapped.status >= http.StatusInternalServerError { + attrs := []any{"request_id", RequestID(c), "error", err.Error()} + if appErr.Cause != nil { + attrs = append(attrs, "cause", appErr.Cause.Error()) + } + logger(c).Error("request failed", attrs...) + } + Fail(c, mapped.status, mapped.code, appErr.Message) +} + +// Timestamp renders RFC 3339 UTC with a fixed millisecond width: Python 3.9 +// (the oldest version the public tests support) only parses 3 or 6 fractional +// digits, and RFC3339Nano would trim trailing zeros to a variable width. +func Timestamp(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z") +} diff --git a/internal/platform/ident/ident.go b/internal/platform/ident/ident.go new file mode 100644 index 00000000..f50f946a --- /dev/null +++ b/internal/platform/ident/ident.go @@ -0,0 +1,56 @@ +// Package ident generates and parses entity identifiers. +// +// Identifiers are UUIDv7 (RFC 9562): the first 48 bits carry a Unix +// millisecond timestamp and google/uuid keeps them strictly increasing within +// one process even when the system clock steps backwards, so lexicographic +// order of ids equals creation order. +package ident + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "time" + + "github.com/google/uuid" +) + +var ErrInvalid = errors.New("invalid uuid") + +// New panics only if the platform random source is unavailable, which is a +// fatal environment fault rather than a request error. +func New() uuid.UUID { + id, err := uuid.NewV7() + if err != nil { + panic("ident: random source unavailable: " + err.Error()) + } + return id +} + +// NewAlias names one server process for X-Instance-Id; random rather than the +// hostname, so the header reveals nothing about the infrastructure. +func NewAlias() string { + var raw [8]byte + _, _ = rand.Read(raw[:]) // crypto/rand.Read never returns an error since Go 1.24. + return hex.EncodeToString(raw[:]) +} + +// TimeOf extracts the creation time carried by a UUIDv7 as UTC with +// millisecond precision. +func TimeOf(id uuid.UUID) time.Time { + sec, nsec := id.Time().UnixTime() + return time.Unix(sec, nsec).UTC() +} + +// Parse accepts only the canonical 36-character form; uuid.Parse alone would +// also take braces, urn prefixes and bare hex. +func Parse(raw string) (uuid.UUID, error) { + if len(raw) != 36 { + return uuid.Nil, ErrInvalid + } + id, err := uuid.Parse(raw) + if err != nil { + return uuid.Nil, ErrInvalid + } + return id, nil +} diff --git a/internal/platform/ident/ident_test.go b/internal/platform/ident/ident_test.go new file mode 100644 index 00000000..d0a57d4d --- /dev/null +++ b/internal/platform/ident/ident_test.go @@ -0,0 +1,84 @@ +package ident_test + +import ( + "bytes" + "regexp" + "testing" + "time" + + "messenger/internal/platform/ident" +) + +func TestNewProducesVersion7(t *testing.T) { + id := ident.New() + if got := id.Version(); got != 7 { + t.Fatalf("version = %d, want 7", got) + } +} + +func TestNewIsStrictlyIncreasingWithinProcess(t *testing.T) { + prev := ident.New() + for i := 0; i < 20000; i++ { + next := ident.New() + if bytes.Compare(next[:], prev[:]) <= 0 { + t.Fatalf("id %s is not greater than previous %s at iteration %d", next, prev, i) + } + prev = next + } +} + +func TestTimeOfMatchesWallClockAtMillisecondPrecision(t *testing.T) { + before := time.Now().Truncate(time.Millisecond) + id := ident.New() + after := time.Now().Add(time.Millisecond) + + got := ident.TimeOf(id) + if got.Location() != time.UTC { + t.Fatalf("TimeOf must return UTC, got %v", got.Location()) + } + if got.Before(before) || got.After(after) { + t.Fatalf("TimeOf = %v, want within [%v, %v]", got, before, after) + } + if got.Nanosecond()%int(time.Millisecond) != 0 { + t.Fatalf("TimeOf must be truncated to milliseconds, got %v", got) + } +} + +func TestParseAcceptsCanonicalForm(t *testing.T) { + id := ident.New() + parsed, err := ident.Parse(id.String()) + if err != nil { + t.Fatalf("Parse(%q) error: %v", id, err) + } + if parsed != id { + t.Fatalf("Parse round trip mismatch: %s != %s", parsed, id) + } +} + +func TestParseRejectsNonCanonicalForms(t *testing.T) { + id := ident.New().String() + for _, raw := range []string{ + "", + "not-a-uuid", + "{" + id + "}", + "urn:uuid:" + id, + id[:32] + id[33:], // hex without one dash + id + "0", + " " + id, + } { + if _, err := ident.Parse(raw); err == nil { + t.Errorf("Parse(%q) must fail", raw) + } + } +} + +func TestNewAliasIs16HexAndRandom(t *testing.T) { + a, b := ident.NewAlias(), ident.NewAlias() + hex16 := regexp.MustCompile(`^[0-9a-f]{16}$`) + if !hex16.MatchString(a) || !hex16.MatchString(b) { + t.Fatalf("aliases %q, %q are not 16 lowercase hex characters", a, b) + } + if a == b { + t.Fatalf("two aliases are equal: %q", a) + } +} diff --git a/internal/platform/pagination/cursor.go b/internal/platform/pagination/cursor.go new file mode 100644 index 00000000..65a09854 --- /dev/null +++ b/internal/platform/pagination/cursor.go @@ -0,0 +1,63 @@ +// Package pagination implements keyset paging. A cursor is opaque to the +// client and bound to the resource it pages (the chat for history, the user +// for the chat list), so a cursor from one resource is rejected on another. +// Because ids are UUIDv7 and creation time derives from the id, the keyset +// (created_at, id) collapses to the id of the last returned item. +package pagination + +import ( + "encoding/base64" + "errors" + "strconv" + + "github.com/google/uuid" +) + +const ( + DefaultLimit = 50 + MaxLimit = 100 +) + +var ( + ErrInvalidCursor = errors.New("invalid cursor") + ErrInvalidLimit = errors.New("limit must be an integer between 1 and 100") +) + +const cursorVersion = 1 + +func EncodeCursor(scope, last uuid.UUID) string { + raw := make([]byte, 0, 1+16+16) + raw = append(raw, cursorVersion) + raw = append(raw, scope[:]...) + raw = append(raw, last[:]...) + return base64.RawURLEncoding.EncodeToString(raw) +} + +func DecodeCursor(token string, scope uuid.UUID) (uuid.UUID, error) { + raw, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(raw) != 1+16+16 || raw[0] != cursorVersion { + return uuid.Nil, ErrInvalidCursor + } + if got, err := uuid.FromBytes(raw[1:17]); err != nil || got != scope { + return uuid.Nil, ErrInvalidCursor + } + last, err := uuid.FromBytes(raw[17:]) + if err != nil { + return uuid.Nil, ErrInvalidCursor + } + return last, nil +} + +// ParseLimit accepts an unsigned decimal between 1 and MaxLimit; an empty +// value means DefaultLimit. ParseUint rejects signs, whitespace and +// fractions and reports overflow, so no length guard is needed. +func ParseLimit(raw string) (int, error) { + if raw == "" { + return DefaultLimit, nil + } + n, err := strconv.ParseUint(raw, 10, 32) + if err != nil || n < 1 || n > MaxLimit { + return 0, ErrInvalidLimit + } + return int(n), nil +} diff --git a/internal/platform/pagination/cursor_test.go b/internal/platform/pagination/cursor_test.go new file mode 100644 index 00000000..5c93a17c --- /dev/null +++ b/internal/platform/pagination/cursor_test.go @@ -0,0 +1,71 @@ +package pagination_test + +import ( + "errors" + "strings" + "testing" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func TestCursorRoundTrip(t *testing.T) { + scope, last := ident.New(), ident.New() + encoded := pagination.EncodeCursor(scope, last) + if encoded == "" || strings.ContainsAny(encoded, "+/=") { + t.Fatalf("cursor must be non-empty URL-safe base64 without padding, got %q", encoded) + } + got, err := pagination.DecodeCursor(encoded, scope) + if err != nil { + t.Fatalf("DecodeCursor error: %v", err) + } + if got != last { + t.Fatalf("decoded %s, want %s", got, last) + } +} + +func TestDecodeCursorRejectsForeignScope(t *testing.T) { + encoded := pagination.EncodeCursor(ident.New(), ident.New()) + _, err := pagination.DecodeCursor(encoded, ident.New()) + if !errors.Is(err, pagination.ErrInvalidCursor) { + t.Fatalf("err = %v, want ErrInvalidCursor", err) + } +} + +func TestDecodeCursorRejectsGarbage(t *testing.T) { + scope := ident.New() + valid := pagination.EncodeCursor(scope, ident.New()) + for _, raw := range []string{"", "***", "abc", valid[:len(valid)-2], valid + "AA", strings.ToUpper(valid)} { + if _, err := pagination.DecodeCursor(raw, scope); !errors.Is(err, pagination.ErrInvalidCursor) { + t.Errorf("DecodeCursor(%q) err = %v, want ErrInvalidCursor", raw, err) + } + } +} + +func TestParseLimit(t *testing.T) { + cases := []struct { + raw string + want int + ok bool + }{ + {"", 50, true}, + {"1", 1, true}, + {"100", 100, true}, + {"0", 0, false}, + {"101", 0, false}, + {"-1", 0, false}, + {"+5", 0, false}, + {"5.0", 0, false}, + {"abc", 0, false}, + {" 5", 0, false}, + } + for _, tc := range cases { + got, err := pagination.ParseLimit(tc.raw) + if tc.ok && (err != nil || got != tc.want) { + t.Errorf("ParseLimit(%q) = %d, %v; want %d", tc.raw, got, err, tc.want) + } + if !tc.ok && !errors.Is(err, pagination.ErrInvalidLimit) { + t.Errorf("ParseLimit(%q) err = %v, want ErrInvalidLimit", tc.raw, err) + } + } +} diff --git a/internal/platform/pagination/page.go b/internal/platform/pagination/page.go new file mode 100644 index 00000000..f905be82 --- /dev/null +++ b/internal/platform/pagination/page.go @@ -0,0 +1,41 @@ +package pagination + +import "github.com/google/uuid" + +// Page is the service-level page: the items and the id to continue from, +// nil when the listing is exhausted. It carries the raw keyset id so that any +// transport can encode the continuation its own way. +type Page[T any] struct { + Items []T + Next *uuid.UUID +} + +// PageResponse is the wire form of a page with the continuation already +// encoded as an opaque cursor. +type PageResponse[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Cut turns a slice over-fetched by one item into a page of at most limit +// items; the extra item only proves that a next page exists. +func Cut[T any](items []T, limit int, id func(T) uuid.UUID) Page[T] { + if len(items) <= limit { + return Page[T]{Items: items} + } + items = items[:limit] + next := id(items[limit-1]) + return Page[T]{Items: items, Next: &next} +} + +func ToResponse[D, R any](p Page[D], scope uuid.UUID, convert func(D) R) PageResponse[R] { + out := PageResponse[R]{Items: make([]R, 0, len(p.Items))} + for _, item := range p.Items { + out.Items = append(out.Items, convert(item)) + } + if p.Next != nil { + cursor := EncodeCursor(scope, *p.Next) + out.NextCursor = &cursor + } + return out +} diff --git a/internal/platform/pagination/page_test.go b/internal/platform/pagination/page_test.go new file mode 100644 index 00000000..6c3b0272 --- /dev/null +++ b/internal/platform/pagination/page_test.go @@ -0,0 +1,60 @@ +package pagination_test + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/pagination" +) + +func self(id uuid.UUID) uuid.UUID { return id } + +func TestCutTrimsOverfetchAndDerivesNext(t *testing.T) { + ids := []uuid.UUID{ident.New(), ident.New(), ident.New()} + + fewer := pagination.Cut(ids[:2], 5, self) + if len(fewer.Items) != 2 || fewer.Next != nil { + t.Fatalf("fewer than limit: %+v", fewer) + } + exact := pagination.Cut(ids[:2], 2, self) + if len(exact.Items) != 2 || exact.Next != nil { + t.Fatalf("exactly limit must not continue: %+v", exact) + } + over := pagination.Cut(ids, 2, self) + if len(over.Items) != 2 || over.Next == nil || *over.Next != ids[1] { + t.Fatalf("over-fetched page must trim and continue from the last kept id: %+v", over) + } + if over.Items[0] != ids[0] || over.Items[1] != ids[1] { + t.Fatalf("order changed: %v", over.Items) + } +} + +func TestToResponseEncodesCursorWithinScope(t *testing.T) { + scope, last := ident.New(), ident.New() + page := pagination.Page[uuid.UUID]{Items: []uuid.UUID{last}, Next: &last} + res := pagination.ToResponse(page, scope, func(id uuid.UUID) string { return id.String() }) + if len(res.Items) != 1 || res.Items[0] != last.String() { + t.Fatalf("items = %v", res.Items) + } + if res.NextCursor == nil { + t.Fatal("next cursor missing") + } + got, err := pagination.DecodeCursor(*res.NextCursor, scope) + if err != nil || got != last { + t.Fatalf("cursor decodes to %s, %v; want %s", got, err, last) + } +} + +func TestToResponseSerializesEmptyPageAsContractRequires(t *testing.T) { + res := pagination.ToResponse(pagination.Page[uuid.UUID]{}, ident.New(), func(id uuid.UUID) string { return id.String() }) + raw, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + if string(raw) != `{"items":[],"next_cursor":null}` { + t.Fatalf("empty page = %s", raw) + } +} diff --git a/internal/platform/password/password.go b/internal/platform/password/password.go new file mode 100644 index 00000000..7c22fdde --- /dev/null +++ b/internal/platform/password/password.go @@ -0,0 +1,74 @@ +// Package password hashes user passwords with Argon2id. +// +// A password has little entropy, so its hash must be slow and memory-hard to +// make offline guessing expensive. Session tokens are the opposite case: they +// are random 256-bit values, and a fast digest is enough for them. +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +// OWASP baseline for Argon2id. The parameters are written into every hash, +// so raising them later does not invalidate stored hashes. +const ( + memoryKiB = 19 * 1024 + iterations = 2 + threads = 1 + saltLen = 16 + keyLen = 32 +) + +var ErrMalformed = errors.New("malformed password hash") + +var b64 = base64.RawStdEncoding + +// Hash returns the PHC string $argon2id$v=19$m=…,t=…,p=…$$. +func Hash(plain string) string { + salt := make([]byte, saltLen) + // Since Go 1.24 crypto/rand.Read never returns an error. + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte(plain), salt, iterations, memoryKiB, threads, keyLen) + return fmt.Sprintf("$argon2id$v=%d$%s$%s$%s", + argon2.Version, params(memoryKiB, iterations, threads), b64.EncodeToString(salt), b64.EncodeToString(key)) +} + +// Verify reports whether plain matches encoded. An error means the stored +// hash itself is unreadable, not that the password is wrong. +func Verify(encoded, plain string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" { + return false, ErrMalformed + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrMalformed + } + var memory, time uint32 + var parallelism uint8 + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, ¶llelism); err != nil || + memory == 0 || time == 0 || parallelism == 0 { + return false, ErrMalformed + } + salt, err := b64.DecodeString(parts[4]) + if err != nil || len(salt) == 0 { + return false, ErrMalformed + } + key, err := b64.DecodeString(parts[5]) + if err != nil || len(key) == 0 { + return false, ErrMalformed + } + got := argon2.IDKey([]byte(plain), salt, time, memory, parallelism, uint32(len(key))) + return subtle.ConstantTimeCompare(got, key) == 1, nil +} + +func params(memory, time uint32, parallelism uint8) string { + return fmt.Sprintf("m=%d,t=%d,p=%d", memory, time, parallelism) +} diff --git a/internal/platform/password/password_test.go b/internal/platform/password/password_test.go new file mode 100644 index 00000000..7c518ad1 --- /dev/null +++ b/internal/platform/password/password_test.go @@ -0,0 +1,78 @@ +package password_test + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "strings" + "testing" + + "golang.org/x/crypto/argon2" + + "messenger/internal/platform/password" +) + +func TestHashVerifiesOnlyTheOriginalPassword(t *testing.T) { + for _, plain := range []string{ + "correct horse battery staple", + "с:двоеточием:и кириллицей", + "emoji 👋👋👋 and : colon", + } { + encoded := password.Hash(plain) + ok, err := password.Verify(encoded, plain) + if err != nil || !ok { + t.Fatalf("Verify(%q) = %v, %v; want true", plain, ok, err) + } + ok, err = password.Verify(encoded, plain+"x") + if err != nil || ok { + t.Fatalf("Verify(wrong) = %v, %v; want false", ok, err) + } + } +} + +func TestHashIsSaltedPHCString(t *testing.T) { + a, b := password.Hash("same password"), password.Hash("same password") + if a == b { + t.Fatal("two hashes of one password are equal: salt is missing") + } + if !strings.HasPrefix(a, "$argon2id$v=19$m=19456,t=2,p=1$") { + t.Fatalf("unexpected encoding %q", a) + } + if strings.Contains(a, "same password") { + t.Fatal("hash contains the plaintext") + } +} + +// A hash made with other parameters must still verify: parameters travel in +// the encoded string, so they can be raised without invalidating old hashes. +func TestVerifyUsesParametersFromTheEncoding(t *testing.T) { + salt := make([]byte, 16) + _, _ = rand.Read(salt) + key := argon2.IDKey([]byte("legacy password"), salt, 1, 8*1024, 2, 32) + b64 := base64.RawStdEncoding + encoded := fmt.Sprintf("$argon2id$v=19$m=8192,t=1,p=2$%s$%s", b64.EncodeToString(salt), b64.EncodeToString(key)) + + ok, err := password.Verify(encoded, "legacy password") + if err != nil || !ok { + t.Fatalf("Verify = %v, %v; want true", ok, err) + } +} + +func TestVerifyRejectsMalformedEncoding(t *testing.T) { + valid := password.Hash("whatever password") + parts := strings.Split(valid, "$") + for name, encoded := range map[string]string{ + "empty": "", + "wrong algo": strings.Replace(valid, "argon2id", "argon2i", 1), + "wrong version": strings.Replace(valid, "v=19", "v=16", 1), + "bad params": strings.Replace(valid, "m=19456,t=2,p=1", "m=x,t=2,p=1", 1), + "zero params": strings.Replace(valid, "m=19456,t=2,p=1", "m=19456,t=0,p=1", 1), + "bad salt": strings.Join([]string{"", parts[1], parts[2], parts[3], "!!!", parts[5]}, "$"), + "bad key": strings.Join([]string{"", parts[1], parts[2], parts[3], parts[4], "!!!"}, "$"), + "missing fields": strings.Join(parts[:5], "$"), + } { + if _, err := password.Verify(encoded, "whatever password"); err == nil { + t.Errorf("%s: Verify accepted %q", name, encoded) + } + } +} diff --git a/internal/platform/postgres/postgres.go b/internal/platform/postgres/postgres.go new file mode 100644 index 00000000..d9a7d653 --- /dev/null +++ b/internal/platform/postgres/postgres.go @@ -0,0 +1,143 @@ +// Package postgres opens the connection pool and applies the schema. +// +// Server-side timeouts are constants rather than settings: they bound how +// long one request can hold a connection, and the request deadline of the API +// is chosen against them. +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/lock" + + "messenger/internal/platform/apperr" + "messenger/migrations" +) + +const ( + connectTimeout = time.Second + statementTimeout = 5 * time.Second + idleInTxTimeout = 10 * time.Second +) + +const codeUniqueViolation = "23505" + +func NewPool(ctx context.Context, dsn string, maxConns int32) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("parse database url: %w", err) + } + cfg.MaxConns = maxConns + cfg.ConnConfig.ConnectTimeout = connectTimeout + // Unlike net.Dialer, pgx resolves the host before ConnectTimeout starts. + // Docker DNS takes seconds to give up on a stopped container: unbounded, + // every request would wait out its deadline and log only "context + // deadline exceeded" instead of the failed lookup. + cfg.ConnConfig.LookupFunc = func(ctx context.Context, host string) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + return net.DefaultResolver.LookupHost(ctx, host) + } + cfg.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(statementTimeout.Milliseconds(), 10) + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = strconv.FormatInt(idleInTxTimeout.Milliseconds(), 10) + // The application keeps every time in UTC; pgx would otherwise scan + // timestamptz into time.Local. + cfg.AfterConnect = func(_ context.Context, conn *pgx.Conn) error { + conn.TypeMap().RegisterType(&pgtype.Type{ + Name: "timestamptz", + OID: pgtype.TimestamptzOID, + Codec: &pgtype.TimestamptzCodec{ScanLocation: time.UTC}, + }) + return nil + } + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("create pool: %w", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + return pool, nil +} + +// Migrate applies pending migrations and returns their versions; a session +// advisory lock makes overlapping runs wait for each other. +func Migrate(ctx context.Context, pool *pgxpool.Pool) ([]int64, error) { + locker, err := lock.NewPostgresSessionLocker() + if err != nil { + return nil, fmt.Errorf("migration lock: %w", err) + } + db := stdlib.OpenDBFromPool(pool) + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.FS, goose.WithSessionLocker(locker)) + if err != nil { + db.Close() + return nil, fmt.Errorf("load migrations: %w", err) + } + defer provider.Close() + results, err := provider.Up(ctx) + if err != nil { + return nil, fmt.Errorf("apply migrations: %w", err) + } + applied := make([]int64, 0, len(results)) + for _, r := range results { + applied = append(applied, r.Source.Version) + } + return applied, nil +} + +// Classify marks a failure to reach the database as apperr.KindUnavailable, +// which the API answers with 503 instead of 500; any other error is returned +// unchanged. Unreachable means: the connection could not be made or broke, +// the request deadline passed, or the server refused work (PostgreSQL error +// classes 08 connection exception, 53 insufficient resources, 57 operator +// intervention such as shutdown or statement timeout). +func Classify(err error) error { + if err == nil || !unreachable(err) { + return err + } + return apperr.Wrap(apperr.KindUnavailable, "database unavailable", err) +} + +func unreachable(err error) bool { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + switch class := pgErr.Code[:min(2, len(pgErr.Code))]; class { + case "08", "53", "57": + return true + } + return false + } + if _, ok := errors.AsType[*pgconn.ConnectError](err); ok { + return true + } + if _, ok := errors.AsType[net.Error](err); ok { + return true + } + return errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, net.ErrClosed) || + pgconn.Timeout(err) || + pgconn.SafeToRetry(err) +} + +// IsUniqueViolation reports whether err is a unique violation of the named +// constraint. +func IsUniqueViolation(err error, constraint string) bool { + pgErr, ok := errors.AsType[*pgconn.PgError](err) + return ok && pgErr.Code == codeUniqueViolation && pgErr.ConstraintName == constraint +} diff --git a/internal/platform/postgres/postgres_test.go b/internal/platform/postgres/postgres_test.go new file mode 100644 index 00000000..6f56fb39 --- /dev/null +++ b/internal/platform/postgres/postgres_test.go @@ -0,0 +1,212 @@ +package postgres_test + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/postgres" + "messenger/internal/testenv" + "messenger/migrations" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} + +func TestPoolScansTimestamptzInUTC(t *testing.T) { + var got time.Time + if err := db.Pool.QueryRow(context.Background(), `SELECT now()`).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Location() != time.UTC { + t.Fatalf("timestamptz scanned in %v, want UTC", got.Location()) + } +} + +func TestMigrateIsRepeatable(t *testing.T) { + ctx := context.Background() + applied, err := postgres.Migrate(ctx, db.Pool) + if err != nil { + t.Fatalf("second migrate: %v", err) + } + if len(applied) != 0 { + t.Fatalf("second migrate applied %v, want nothing", applied) + } + var version int64 + if err := db.Pool.QueryRow(ctx, `SELECT max(version_id) FROM goose_db_version`).Scan(&version); err != nil { + t.Fatal(err) + } + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + if version != int64(len(files)) { + t.Fatalf("schema version = %d, want %d (one per migration file)", version, len(files)) + } + for _, table := range []string{"users", "chats", "chat_members", "messages", "message_idempotency_keys"} { + var exists bool + if err := db.Pool.QueryRow(ctx, `SELECT to_regclass('public.' || $1) IS NOT NULL`, table).Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("table %s is missing", table) + } + } +} + +func TestPoolAppliesLimitsAndTimeouts(t *testing.T) { + if got := db.Pool.Config().MaxConns; got != 10 { + t.Fatalf("MaxConns = %d, want 10", got) + } + if got := db.Pool.Config().ConnConfig.ConnectTimeout; got != time.Second { + t.Fatalf("ConnectTimeout = %v, want 1s", got) + } + ctx := context.Background() + for setting, want := range map[string]string{ + "statement_timeout": "5s", + "idle_in_transaction_session_timeout": "10s", + } { + var got string + if err := db.Pool.QueryRow(ctx, "SHOW "+setting).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("%s = %q, want %q", setting, got, want) + } + } +} + +func TestResetEmptiesEveryTableButKeepsSchemaVersion(t *testing.T) { + ctx := context.Background() + _, err := db.Pool.Exec(ctx, ` + WITH u AS ( + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000001', 'reset_check', 'Reset', 'x', now()) + RETURNING id + ), c AS ( + INSERT INTO chats (id, title, created_at) + VALUES ('0190f1d2-0000-7000-8000-000000000002', 'Reset', now()) + RETURNING id + ), m AS ( + INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM c, u + ) + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT '0190f1d2-0000-7000-8000-000000000003', c.id, u.id, 'hi', now() FROM c, u`) + if err != nil { + t.Fatal(err) + } + + db.Reset(t) + + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + var n int + if err := db.Pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%s has %d rows after Reset", table, n) + } + } + var versions int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM goose_db_version`).Scan(&versions); err != nil { + t.Fatal(err) + } + if versions == 0 { + t.Fatal("Reset wiped the schema version table") + } +} + +func TestIsUniqueViolation(t *testing.T) { + db.Reset(t) + ctx := context.Background() + insert := `INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, 'taken', 'Taken', 'x', now())` + if _, err := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000011"); err != nil { + t.Fatal(err) + } + _, dup := db.Pool.Exec(ctx, insert, "0190f1d2-0000-7000-8000-000000000012") + if dup == nil { + t.Fatal("duplicate username was accepted") + } + + if !postgres.IsUniqueViolation(dup, "users_username_key") { + t.Fatalf("not recognised as a username violation: %v", dup) + } + if postgres.IsUniqueViolation(dup, "some_other_key") { + t.Fatal("matched a different constraint") + } + if postgres.IsUniqueViolation(fmt.Errorf("wrapped: %w", dup), "users_username_key") == false { + t.Fatal("wrapped error not recognised") + } + if postgres.IsUniqueViolation(errors.New("plain"), "users_username_key") || postgres.IsUniqueViolation(nil, "users_username_key") { + t.Fatal("non-database error recognised as a violation") + } +} + +func TestClassifyMarksUnreachableDatabaseAsUnavailable(t *testing.T) { + unavailable := map[string]error{ + "admin shutdown": &pgconn.PgError{Code: "57P01"}, + "statement timeout": &pgconn.PgError{Code: "57014"}, + "connection failure": &pgconn.PgError{Code: "08006"}, + "too many connections": &pgconn.PgError{Code: "53300"}, + "deadline": fmt.Errorf("query: %w", context.DeadlineExceeded), + "network": &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")}, + "unexpected eof": fmt.Errorf("read: %w", io.ErrUnexpectedEOF), + } + for name, err := range unavailable { + got := postgres.Classify(err) + if apperr.KindOf(got) != apperr.KindUnavailable || !errors.Is(got, err) { + t.Errorf("%s: Classify = %v (kind %v), want KindUnavailable keeping the cause", name, got, apperr.KindOf(got)) + } + } + for name, err := range map[string]error{ + "unique violation": &pgconn.PgError{Code: "23505"}, + "plain": errors.New("boom"), + "nil": nil, + } { + if got := postgres.Classify(err); got != err { + t.Errorf("%s: Classify = %v, want the error unchanged", name, got) + } + } +} + +func TestNewPoolUsesTheRequestedSize(t *testing.T) { + pool, err := postgres.NewPool(context.Background(), db.DSN, 3) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if got := pool.Config().MaxConns; got != 3 { + t.Fatalf("MaxConns = %d, want 3", got) + } +} + +func TestClassifyRealConnectFailure(t *testing.T) { + _, err := postgres.NewPool(context.Background(), "postgres://u:p@127.0.0.1:1/db?connect_timeout=1", 1) + if err == nil { + t.Fatal("NewPool to a closed port must fail") + } + if apperr.KindOf(postgres.Classify(err)) != apperr.KindUnavailable { + t.Fatalf("connect failure classified as %v: %v", apperr.KindOf(postgres.Classify(err)), err) + } +} diff --git a/internal/platform/s3store/bootstrap_test.go b/internal/platform/s3store/bootstrap_test.go new file mode 100644 index 00000000..d68429ff --- /dev/null +++ b/internal/platform/s3store/bootstrap_test.go @@ -0,0 +1,143 @@ +package s3store_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" +) + +func TestBootstrapWaitsForRetainedBytesAndCleansWriteProbe(t *testing.T) { + var ready atomic.Bool + var reads atomic.Int32 + var mu sync.Mutex + objects := map[string]string{".health/storage-v1": "messenger s3 readiness v1\n"} + var freshReads, deleted int + requested := make(chan struct{}, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + if r.Method == "GET" && key == ".health/storage-v1" { + reads.Add(1) + if !ready.Load() { + select { + case requested <- struct{}{}: + default: + } + w.WriteHeader(500) + io.WriteString(w, `InternalErrorvolume not registered`) + return + } + } + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "GET": + value, ok := objects[key] + if !ok { + w.WriteHeader(404) + io.WriteString(w, `NoSuchKey`) + return + } + if key != ".health/storage-v1" { + freshReads++ + } + w.Header().Set("ETag", `"probe"`) + w.Header().Set("Last-Modified", "Mon, 16 Nov 2026 08:00:00 GMT") + w.Header().Set("Content-Length", fmt.Sprint(len(value))) + io.WriteString(w, value) + case "PUT": + var body io.Reader = r.Body + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") { + body = httputil.NewChunkedReader(body) + } + data, err := io.ReadAll(io.LimitReader(body, 1024)) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + objects[key] = string(data) + w.Header().Set("ETag", `"probe"`) + case "DELETE": + if key == ".health/storage-v1" { + t.Error("deleted retained readiness marker") + } + delete(objects, key) + deleted++ + w.WriteHeader(204) + default: + t.Errorf("unexpected storage probe: %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer server.Close() + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- s.Bootstrap(ctx) }() + select { + case err := <-done: + t.Fatalf("bootstrap returned before reading retained storage: %v", err) + case <-requested: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case err := <-done: + t.Fatalf("bootstrap accepted unavailable retained storage: %v", err) + case <-time.After(40 * time.Millisecond): + } + ready.Store(true) + if err := <-done; err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if reads.Load() < 2 || freshReads == 0 || deleted == 0 || len(objects) != 1 { + t.Fatalf("incomplete readiness proof: retainedReads=%d freshReads=%d deletes=%d remaining=%d", reads.Load(), freshReads, deleted, len(objects)) + } +} + +func TestBootstrapDeadlineIncludesStorageReadiness(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" && strings.TrimSuffix(r.URL.Path, "/") == "/attachments" { + w.WriteHeader(200) + return + } + w.WriteHeader(503) + io.WriteString(w, `ServiceUnavailablevolume not registered`) + })) + defer server.Close() + s, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + if err := s.Bootstrap(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("storage readiness escaped its deadline: %v", err) + } + if time.Since(started) > time.Second { + t.Fatal("readiness did not honor caller deadline") + } +} diff --git a/internal/platform/s3store/integration_test.go b/internal/platform/s3store/integration_test.go new file mode 100644 index 00000000..4015511b --- /dev/null +++ b/internal/platform/s3store/integration_test.go @@ -0,0 +1,271 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "messenger/internal/attachment" + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" + "messenger/internal/testenv" +) + +func TestSeaweedStore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + newClient := func(access, secret string) *minio.Client { + t.Helper() + c, err := minio.New(strings.TrimPrefix(s.Endpoint, "http://"), &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Region: s.Region, BucketLookup: minio.BucketLookupPath, MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return c + } + api := newClient(s.APIAccessKey, s.APISecretKey) + admin := newClient(s.AdminAccessKey, s.AdminSecretKey) + denied := func(t *testing.T, err error) { + t.Helper() + var response minio.ErrorResponse + if !errors.As(err, &response) || response.StatusCode != http.StatusForbidden { + t.Fatalf("expected access denied, got %v", err) + } + } + t.Run("remote gRPC cannot modify identity cache", func(t *testing.T) { + conn, err := grpc.NewClient("passthrough:///"+s.GRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) { + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, "tcp", address) + })) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + opCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + err = conn.Invoke(opCtx, "/messaging_pb.SeaweedS3IamCache/PutIdentity", &emptypb.Empty{}, &emptypb.Empty{}) + if code := status.Code(err); code != codes.PermissionDenied && code != codes.Unauthenticated { + t.Fatalf("remote IAM mutation was not rejected before argument validation: %v", err) + } + }) + t.Run("bootstrap is idempotent and API cannot administer", func(t *testing.T) { + if err := s.Admin.Bootstrap(ctx); err != nil { + t.Fatal(err) + } + denied(t, s.APIStore.Bootstrap(ctx)) + denied(t, api.MakeBucket(ctx, "forbidden-bucket", minio.MakeBucketOptions{Region: s.Region})) + denied(t, api.RemoveBucket(ctx, s.Bucket)) + buckets, err := api.ListBuckets(ctx) + // SeaweedFS returns a filtered empty result for an authenticated caller + // without List rights, rather than rejecting ListBuckets outright. + if err != nil || len(buckets) != 0 { + t.Fatalf("API enumerated buckets: count=%d error=%v", len(buckets), err) + } + _, err = s.APIStore.List(ctx, "", "", 1) + denied(t, err) + }) + t.Run("known length nonseekable put copy stat and stream", func(t *testing.T) { + const payload = "nonseekable image payload" + r := struct{ io.Reader }{strings.NewReader(payload)} + started := time.Now() + if err := s.APIStore.Put(ctx, "staging/source", r, int64(len(payload)), "image/png"); err != nil { + t.Fatal(err) + } + t.Logf("cold first PUT: %s", time.Since(started)) + if err := s.APIStore.Copy(ctx, "staging/source", "blobs/durable"); err != nil { + t.Fatal(err) + } + obj, err := s.APIStore.Stat(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + if obj.Key != "blobs/durable" || obj.Size != 25 || obj.Modified.IsZero() { + t.Fatalf("wrong metadata: %+v", obj) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != payload { + t.Fatalf("wrong body: %q", b) + } + meta, err := api.StatObject(ctx, s.Bucket, "blobs/durable", minio.StatObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if meta.ContentType != "image/png" { + t.Fatalf("copy lost MIME: %s", meta.ContentType) + } + for range 2 { + if err := s.APIStore.Delete(ctx, "staging/source"); err != nil { + t.Fatal(err) + } + } + if _, err := s.APIStore.Stat(ctx, "staging/source"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("API delete did not remove staging: %v", err) + } + }) + t.Run("anonymous get put head and listing are refused", func(t *testing.T) { + client := &http.Client{Timeout: 3 * time.Second} + defer client.CloseIdleConnections() + for _, tc := range []struct{ method, path string }{{"GET", "/"}, {"GET", "/" + s.Bucket + "?list-type=2"}, {"GET", "/" + s.Bucket + "/blobs/durable"}, {"HEAD", "/" + s.Bucket + "/blobs/durable"}, {"PUT", "/" + s.Bucket + "/anonymous"}} { + req, _ := http.NewRequestWithContext(ctx, tc.method, s.Endpoint+tc.path, strings.NewReader("x")) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("anonymous %s %s: %d", tc.method, tc.path, resp.StatusCode) + } + } + }) + t.Run("identities cannot escape their bucket", func(t *testing.T) { + if err := admin.MakeBucket(ctx, "private-other", minio.MakeBucketOptions{Region: s.Region}); err != nil { + t.Fatal(err) + } + if _, err := admin.PutObject(ctx, "private-other", "secret", strings.NewReader("private"), 7, minio.PutObjectOptions{DisableMultipart: true}); err != nil { + t.Fatal(err) + } + for _, pair := range [][2]string{{s.APIAccessKey, s.APISecretKey}, {s.AccessKey, s.SecretKey}} { + outside, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "private-other", AccessKey: pair[0], SecretKey: pair[1]}) + if err != nil { + t.Fatal(err) + } + _, err = outside.Stat(ctx, "secret") + denied(t, err) + _, err = outside.Open(ctx, "secret") + denied(t, err) + denied(t, outside.Put(ctx, "secret", strings.NewReader("x"), 1, "image/png")) + denied(t, outside.Delete(ctx, "secret")) + _, err = outside.List(ctx, "", "", 1) + denied(t, err) + c := newClient(pair[0], pair[1]) + _, err = c.CopyObject(ctx, minio.CopyDestOptions{Bucket: s.Bucket, Object: "blobs/stolen"}, minio.CopySrcOptions{Bucket: "private-other", Object: "secret"}) + denied(t, err) + } + }) + t.Run("exclusive lexicographic pages including absent cursor", func(t *testing.T) { + for _, key := range []string{"page/z", "page/a", "page/a/nested", "page/b", "page/c"} { + if err := s.Store.Put(ctx, key, strings.NewReader("x"), 1, "image/png"); err != nil { + t.Fatal(err) + } + } + var got []string + after := "" + for { + page, err := s.Store.List(ctx, "page/", after, 2) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + for _, obj := range page { + if obj.Size != 1 || obj.Modified.IsZero() { + t.Fatalf("wrong list metadata: %+v", obj) + } + got = append(got, obj.Key) + } + after = page[len(page)-1].Key + if len(got) > 5 { + t.Fatal("pagination did not advance") + } + } + if strings.Join(got, ",") != "page/a,page/a/nested,page/b,page/c,page/z" { + t.Fatalf("wrong order: %v", got) + } + page, err := s.Store.List(ctx, "page/", "page/az", 2) + if err != nil { + t.Fatal(err) + } + if len(page) != 2 || page[0].Key != "page/b" || page[1].Key != "page/c" { + t.Fatalf("absent cursor: %+v", page) + } + }) + t.Run("missing objects differ from bucket and permission failures", func(t *testing.T) { + if _, err := s.Store.Stat(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("stat missing: %v", err) + } + if _, err := s.Store.Open(ctx, "absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("open missing: %v", err) + } + if err := s.Store.Copy(ctx, "absent", "blobs/absent"); !errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("copy missing: %v", err) + } + for range 2 { + if err := s.Store.Delete(ctx, "absent"); err != nil { + t.Fatal(err) + } + } + missingBucket, err := s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: "absent-bucket", AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) + if err != nil { + t.Fatal(err) + } + if _, err := missingBucket.Stat(ctx, "absent"); err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("missing bucket treated as missing object: %v", err) + } + }) + t.Run("short readers are not successful uploads", func(t *testing.T) { + if err := s.Store.Put(ctx, "staging/short", struct{ io.Reader }{strings.NewReader("short")}, 100, "image/png"); err == nil { + t.Fatal("short body accepted") + } + }) + t.Run("container replacement preserves volume data and authentication", func(t *testing.T) { + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + body, err := s.APIStore.Open(ctx, "blobs/durable") + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(body) + body.Close() + if err != nil { + t.Fatal(err) + } + if string(b) != "nonseekable image payload" { + t.Fatalf("object lost on replacement: %q", b) + } + if _, err := s.Store.List(ctx, "page/", "", 2); err != nil { + t.Fatal(err) + } + }) + t.Run("outage is bounded and never missing", func(t *testing.T) { + if err := s.Stop(ctx); err != nil { + t.Fatal(err) + } + opCtx, stop := context.WithTimeout(ctx, 150*time.Millisecond) + defer stop() + start := time.Now() + _, err := s.Store.Stat(opCtx, "blobs/durable") + if err == nil || errors.Is(err, attachment.ErrObjectMissing) { + t.Fatalf("outage treated as missing: %v", err) + } + if time.Since(start) > 2*time.Second { + t.Fatal("outage exceeded caller deadline") + } + }) +} diff --git a/internal/platform/s3store/readiness.go b/internal/platform/s3store/readiness.go new file mode 100644 index 00000000..1f6830e3 --- /dev/null +++ b/internal/platform/s3store/readiness.go @@ -0,0 +1,114 @@ +package s3store + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/minio/minio-go/v7" + + "messenger/internal/attachment" +) + +const ( + storageMarkerKey = ".health/storage-v1" + storageMarkerBody = "messenger s3 readiness v1\n" +) + +var errStorageMarker = errors.New("s3: storage readiness marker has unexpected contents") + +// WaitReady is a startup barrier, deliberately separate from API readiness and +// ordinary object operations. Gateway authentication and bucket metadata can be +// available before the master elects its leader and registers retained volumes. +// Read the persistent marker before writing anything, then prove new writes, +// reads and deletes work. The marker lives outside the reconciler's namespaces. +func (s *Store) WaitReady(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + readyCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + probeKey := ".health/check-" + rand.Text() + dirty := false + defer func() { + if dirty { + cleanupCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + defer stop() + _ = s.Delete(cleanupCtx, probeKey) + } + }() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + err := s.checkStorage(readyCtx, probeKey, &dirty) + if err == nil { + return nil + } + if permanentStorageError(err) { + return err + } + select { + case <-readyCtx.Done(): + return fmt.Errorf("s3: wait for readable and writable storage: %w (last probe: %v)", readyCtx.Err(), err) + case <-ticker.C: + } + } +} + +func (s *Store) checkStorage(ctx context.Context, probeKey string, dirty *bool) error { + err := s.readStorageMarker(ctx, storageMarkerKey) + if errors.Is(err, attachment.ErrObjectMissing) { + // Only a genuinely absent marker permits its first creation. Never + // overwrite retained data to hide an unavailable volume or a bad marker. + if err := s.Put(ctx, storageMarkerKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + err = s.readStorageMarker(ctx, storageMarkerKey) + } + if err != nil { + return err + } + *dirty = true + if err := s.Put(ctx, probeKey, strings.NewReader(storageMarkerBody), int64(len(storageMarkerBody)), "application/octet-stream"); err != nil { + return err + } + if err := s.readStorageMarker(ctx, probeKey); err != nil { + return err + } + if err := s.Delete(ctx, probeKey); err != nil { + return err + } + *dirty = false + return nil +} + +func (s *Store) readStorageMarker(ctx context.Context, key string) error { + r, err := s.Open(ctx, key) + if err != nil { + return err + } + data, readErr := io.ReadAll(io.LimitReader(r, int64(len(storageMarkerBody))+1)) + closeErr := r.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + if string(data) != storageMarkerBody { + return errStorageMarker + } + return nil +} + +func permanentStorageError(err error) bool { + if errors.Is(err, errStorageMarker) { + return true + } + var response minio.ErrorResponse + return errors.As(err, &response) && response.StatusCode >= 400 && response.StatusCode < 500 && response.StatusCode != 408 && response.StatusCode != 429 +} diff --git a/internal/platform/s3store/readiness_test.go b/internal/platform/s3store/readiness_test.go new file mode 100644 index 00000000..c92c5fb2 --- /dev/null +++ b/internal/platform/s3store/readiness_test.go @@ -0,0 +1,48 @@ +package s3store_test + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "messenger/internal/testenv" +) + +func TestRecreateReturnsWithRetainedObjectReadable(t *testing.T) { + // Three independent server replacements each have the production 60-second + // startup budget; no per-request transport timeout is changed by this test. + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + s, err := testenv.StartS3(ctx) + if err != nil { + t.Fatal(err) + } + defer s.Close() + defer func() { + if t.Failed() { + t.Log(s.Diagnostics(context.Background())) + } + }() + const payload = "retained storage readiness proof" + if err := s.Store.Put(ctx, "blobs/retained", strings.NewReader(payload), int64(len(payload)), "application/octet-stream"); err != nil { + t.Fatal(err) + } + for attempt := range 3 { + started := time.Now() + if err := s.Recreate(ctx); err != nil { + t.Fatal(err) + } + r, err := s.Store.Open(ctx, "blobs/retained") + if err != nil { + t.Fatalf("replacement %d returned before retained bytes were readable (%s): %v", attempt+1, time.Since(started), err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil || string(data) != payload { + t.Fatalf("replacement %d lost retained bytes: %q %v", attempt+1, data, err) + } + t.Logf("replacement %d ready and retained bytes verified in %s", attempt+1, time.Since(started)) + } +} diff --git a/internal/platform/s3store/store.go b/internal/platform/s3store/store.go new file mode 100644 index 00000000..86d1bb0f --- /dev/null +++ b/internal/platform/s3store/store.go @@ -0,0 +1,201 @@ +package s3store + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + + "messenger/internal/attachment" + "messenger/internal/platform/config" +) + +const MaxPageSize = 500 + +const codeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" + +type Store struct { + client *minio.Core + region string + bucket string +} + +var _ attachment.Blobs = (*Store)(nil) + +// New validates configuration without contacting S3, so text-only service startup +// and readiness do not depend on object storage being reachable. +func New(cfg config.S3) (*Store, error) { + u, err := url.Parse(cfg.Endpoint) + if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return nil, errors.New("s3: endpoint must be an http(s) origin without credentials, query or fragment") + } + if strings.TrimSpace(cfg.Region) == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, errors.New("s3: region and credentials are required") + } + if err := s3utils.CheckValidBucketNameStrict(cfg.Bucket); err != nil { + return nil, fmt.Errorf("s3: invalid bucket: %w", err) + } + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: time.Second, KeepAlive: 30 * time.Second}).DialContext, + TLSHandshakeTimeout: time.Second, + // The first write can wait for the single master's election and volume + // creation. The caller's shorter context still takes precedence. + ResponseHeaderTimeout: 30 * time.Second, + ExpectContinueTimeout: time.Second, + IdleConnTimeout: 30 * time.Second, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 32, + MaxConnsPerHost: 64, + DisableCompression: true, + } + client, err := minio.NewCore(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: u.Scheme == "https", + Region: cfg.Region, + BucketLookup: minio.BucketLookupPath, + Transport: transport, + // A consumed upload stream cannot be replayed. Let the caller decide + // whether to retry a whole operation within its own deadline. + MaxRetries: 1, + }) + if err != nil { + return nil, fmt.Errorf("s3: client: %w", err) + } + return &Store{client: client, region: cfg.Region, bucket: cfg.Bucket}, nil +} + +// Bootstrap is used only by the admin init command, never by API startup. +func (s *Store) Bootstrap(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: s.region}) + code := minio.ToErrorResponse(err).Code + if err != nil && code != codeBucketAlreadyOwnedByYou { + return wrap("create bucket", err) + } + return s.WaitReady(ctx) +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + if err := ctx.Err(); err != nil { + return err + } + if size < 0 || r == nil { + return errors.New("s3: PUT requires a reader and known nonnegative length") + } + _, err := s.client.PutObject(ctx, s.bucket, key, r, size, "", "", minio.PutObjectOptions{ + ContentType: mime, + DisableMultipart: true, + }) + return wrap("put object", err) +} + +func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Core.GetObject performs GET eagerly and returns the streaming body; + // Client.GetObject is lazy and would defer missing/access errors to Read. + r, _, _, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, wrap("open object", err) + } + return r, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (attachment.Object, error) { + if err := ctx.Err(); err != nil { + return attachment.Object{}, err + } + obj, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + // HEAD has no XML error body. The SDK synthesizes NoSuchKey for any + // object-path 404, including a missing bucket; confirm the bucket first. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + exists, bucketErr := s.client.BucketExists(ctx, s.bucket) + if bucketErr != nil { + return attachment.Object{}, wrap("stat bucket", bucketErr) + } + if !exists { + return attachment.Object{}, errors.New("s3: stat object: bucket does not exist") + } + } + return attachment.Object{}, wrap("stat object", err) + } + return attachment.Object{Key: key, Size: obj.Size, Modified: obj.LastModified}, nil +} + +func (s *Store) Copy(ctx context.Context, source, dest string) error { + if err := ctx.Err(); err != nil { + return err + } + _, err := s.client.Client.CopyObject(ctx, + minio.CopyDestOptions{Bucket: s.bucket, Object: dest}, + minio.CopySrcOptions{Bucket: s.bucket, Object: source}, + ) + return wrap("copy object", err) +} + +func (s *Store) Delete(ctx context.Context, key string) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) + // S3 DELETE is idempotent even when the object no longer exists. + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return wrap("delete object", err) +} + +// List returns at most limit keys in byte-lexicographic order, strictly after +// after. The synchronous iterator stops before fetching the next page. +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]attachment.Object, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit < 1 || limit > MaxPageSize { + return nil, fmt.Errorf("s3: list limit must be between 1 and %d", MaxPageSize) + } + listCtx, cancel := context.WithCancel(ctx) + defer cancel() + objects := make([]attachment.Object, 0, limit) + for obj := range s.client.ListObjectsIter(listCtx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, StartAfter: after, Recursive: true, MaxKeys: limit, + }) { + if obj.Err != nil { + return nil, wrap("list objects", obj.Err) + } + objects = append(objects, attachment.Object{Key: obj.Key, Size: obj.Size, Modified: obj.LastModified}) + if len(objects) == limit { + break + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + return objects, nil +} + +func wrap(op string, err error) error { + if err == nil { + return nil + } + // Never treat authorization errors, a missing bucket, or an outage as a + // missing blob: the reconciler would otherwise mark valid files failed. + if code := minio.ToErrorResponse(err).Code; code == "NoSuchKey" || code == "NoSuchObject" { + return fmt.Errorf("s3: %s: %w", op, attachment.ErrObjectMissing) + } + return fmt.Errorf("s3: %s: %w", op, err) +} diff --git a/internal/platform/s3store/store_test.go b/internal/platform/s3store/store_test.go new file mode 100644 index 00000000..41fb2383 --- /dev/null +++ b/internal/platform/s3store/store_test.go @@ -0,0 +1,172 @@ +package s3store_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" +) + +func TestNewIsOfflineAndRejectsInvalidConfiguration(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(503) })) + defer server.Close() + if _, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatal("constructor performed network I/O") + } + for _, tc := range []struct{ name, endpoint, region, bucket, access, secret string }{ + {"scheme", "ftp://localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"path", "http://localhost:8333/path", "us-east-1", "attachments", "access", "secret"}, + {"userinfo", "http://user:password@localhost:8333", "us-east-1", "attachments", "access", "secret"}, + {"query", "http://localhost:8333?x=1", "us-east-1", "attachments", "access", "secret"}, + {"fragment", "http://localhost:8333/#x", "us-east-1", "attachments", "access", "secret"}, + {"region", server.URL, "", "attachments", "access", "secret"}, + {"bucket", server.URL, "us-east-1", "INVALID", "access", "secret"}, + {"access", server.URL, "us-east-1", "attachments", "", "secret"}, + {"secret", server.URL, "us-east-1", "attachments", "access", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := s3store.New(config.S3{Endpoint: tc.endpoint, Region: tc.region, Bucket: tc.bucket, AccessKey: tc.access, SecretKey: tc.secret}); err == nil { + t.Fatal("accepted invalid configuration") + } + }) + } +} + +func TestPutStreamsWithoutReadingTheWholeBodyFirst(t *testing.T) { + first := make(chan struct{}) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodPut || r.URL.Path != "/attachments/staging/stream" || r.URL.RawQuery != "" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(400) + return + } + if r.ContentLength <= 0 || len(r.TransferEncoding) != 0 { + t.Error("PUT is not a known-length single request") + } + b := make([]byte, 1) + if _, err := io.ReadFull(r.Body, b); err != nil { + t.Error(err) + return + } + close(first) + if _, err := io.Copy(io.Discard, r.Body); err != nil { + t.Error(err) + return + } + w.Header().Set("ETag", `"probe"`) + })) + defer server.Close() + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // One signing chunk can be buffered; the remaining bytes wait for the server. + r, w := io.Pipe() + defer r.Close() + go func() { + defer w.Close() + if _, err := w.Write(make([]byte, 256*1024)); err != nil { + return + } + select { + case <-first: + case <-ctx.Done(): + _ = w.CloseWithError(ctx.Err()) + return + } + _, _ = w.Write(make([]byte, 256*1024)) + }() + if err := store.Put(ctx, "staging/stream", r, 512*1024, "image/png"); err != nil { + t.Fatal(err) + } + if requests.Load() != 1 { + t.Fatalf("got %d requests, want one PUT", requests.Load()) + } +} + +func TestCanceledAndInvalidCallsNeverReachTheServer(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + defer server.Close() + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for name, call := range map[string]func() error{ + "put": func() error { return store.Put(ctx, "key", strings.NewReader("x"), 1, "image/png") }, + "open": func() error { + r, e := store.Open(ctx, "key") + if r != nil { + r.Close() + } + return e + }, + "stat": func() error { _, e := store.Stat(ctx, "key"); return e }, + "copy": func() error { return store.Copy(ctx, "a", "b") }, + "delete": func() error { return store.Delete(ctx, "key") }, + "list": func() error { _, e := store.List(ctx, "", "", 1); return e }, + "bootstrap": func() error { return store.Bootstrap(ctx) }, + } { + t.Run(name, func(t *testing.T) { + if err := call(); !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want canceled", err) + } + }) + } + if err := store.Put(context.Background(), "key", strings.NewReader("x"), -1, "image/png"); err == nil { + t.Fatal("accepted unknown length") + } + for _, limit := range []int{-1, 0, 501, 1000000000} { + if _, err := store.List(context.Background(), "", "", limit); err == nil { + t.Fatalf("accepted unbounded page %d", limit) + } + } + if requests.Load() != 0 { + t.Fatalf("invalid calls reached server %d times", requests.Load()) + } +} + +func TestListStopsAfterOneBoundedPage(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Query().Get("start-after") != "blobs/a" || r.URL.Query().Get("prefix") != "blobs/" || r.URL.Query().Get("max-keys") != "2" { + t.Errorf("wrong list query: %s", r.URL.RawQuery) + } + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, `attachmentstruenext-pageblobs/b12026-01-01T00:00:00Zblobs/c22026-01-01T00:00:01Z`) + })) + defer server.Close() + store, err := s3store.New(config.S3{Endpoint: server.URL, Region: "us-east-1", Bucket: "attachments", AccessKey: "access", SecretKey: "secret"}) + if err != nil { + t.Fatal(err) + } + objects, err := store.List(context.Background(), "blobs/", "blobs/a", 2) + if err != nil { + t.Fatal(err) + } + if len(objects) != 2 || objects[0].Key != "blobs/b" || objects[1].Key != "blobs/c" { + t.Fatalf("wrong page: %+v", objects) + } + if requests.Load() != 1 { + t.Fatalf("prefetched %d pages", requests.Load()) + } +} diff --git a/internal/platform/valkey/valkey.go b/internal/platform/valkey/valkey.go new file mode 100644 index 00000000..70c5ba5d --- /dev/null +++ b/internal/platform/valkey/valkey.go @@ -0,0 +1,49 @@ +// Package valkey connects to the Valkey session store. +// +// Timeouts are constants: every authenticated request reads a session, so +// they bound how long a request can wait when the store is slow. Command +// retries are disabled and a connection is dialled once for the same reason: +// a request that cannot reach the store fails fast with 503 instead of +// waiting through a backoff. On the compose stand Docker DNS takes seconds +// to give up on a stopped container, so one dial attempt is already bounded +// only by dialTimeout. +package valkey + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + dialTimeout = time.Second + ioTimeout = time.Second + poolSize = 10 + noRetries = -1 + singleAttempt = 1 +) + +// NewClient connects to url (redis:// scheme) and pings the server. Errors +// never include the URL, which carries the password. +func NewClient(ctx context.Context, url string) (*redis.Client, error) { + opts, err := redis.ParseURL(url) + if err != nil { + return nil, errors.New("session store url is invalid") + } + opts.DialTimeout = dialTimeout + opts.ReadTimeout = ioTimeout + opts.WriteTimeout = ioTimeout + opts.PoolSize = poolSize + opts.MaxRetries = noRetries + opts.DialerRetries = singleAttempt + + client := redis.NewClient(opts) + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, fmt.Errorf("ping session store at %s: %w", opts.Addr, err) + } + return client, nil +} diff --git a/internal/platform/valkey/valkey_test.go b/internal/platform/valkey/valkey_test.go new file mode 100644 index 00000000..efb43ebf --- /dev/null +++ b/internal/platform/valkey/valkey_test.go @@ -0,0 +1,57 @@ +package valkey_test + +import ( + "context" + "strings" + "testing" + "time" + + "messenger/internal/platform/valkey" + "messenger/internal/testenv" +) + +func TestNewClientPingsAndBoundsWaiting(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + client, err := valkey.NewClient(ctx, kv.URL) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer client.Close() + opts := client.Options() + if opts.DialTimeout != time.Second || opts.ReadTimeout != time.Second || opts.WriteTimeout != time.Second { + t.Fatalf("timeouts = %v/%v/%v, want 1s each", opts.DialTimeout, opts.ReadTimeout, opts.WriteTimeout) + } + // go-redis dials up to five times by default; with Docker DNS taking + // seconds to give up on a stopped container that alone would exceed the + // request deadline. + if opts.DialerRetries != 1 { + t.Fatalf("DialerRetries = %d, want a single dial attempt", opts.DialerRetries) + } + // go-redis stores the requested -1 ("no retries") as 0 after init; its + // default would be 3. + if opts.PoolSize != 10 || opts.MaxRetries != 0 { + t.Fatalf("PoolSize = %d, MaxRetries = %d; want 10 and 0 (fail fast)", opts.PoolSize, opts.MaxRetries) + } +} + +func TestNewClientErrorsNeverEchoTheURL(t *testing.T) { + ctx := context.Background() + for name, url := range map[string]string{ + "malformed": "redis://:s3cret@host:port/x", + "unreachable": "redis://:s3cret@127.0.0.1:1/0", + } { + _, err := valkey.NewClient(ctx, url) + if err == nil { + t.Fatalf("%s: want error", name) + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("%s: error leaks the password: %v", name, err) + } + } +} diff --git a/internal/processing/barrier.go b/internal/processing/barrier.go new file mode 100644 index 00000000..5b8155e6 --- /dev/null +++ b/internal/processing/barrier.go @@ -0,0 +1,18 @@ +package processing + +import ( + "context" + "fmt" + "path/filepath" + + "messenger/internal/pipeline" + "messenger/internal/platform/faultbarrier" +) + +// Only the explicit fault experiment enables these one-shot barriers. +func FileBarrier(dir string) func(context.Context, pipeline.Job, string) error { + return func(ctx context.Context, j pipeline.Job, phase string) error { + base := filepath.Join(dir, fmt.Sprintf("%s-v%d-%s", j.AttachmentID, j.PipelineVersion, phase)) + return faultbarrier.Wait(ctx, base, "") + } +} diff --git a/internal/processing/dispatcher.go b/internal/processing/dispatcher.go new file mode 100644 index 00000000..725a3cc5 --- /dev/null +++ b/internal/processing/dispatcher.go @@ -0,0 +1,73 @@ +package processing + +import ( + "context" + "encoding/json" + "log/slog" + "time" + + "messenger/internal/broker" + "messenger/internal/pipeline" + "messenger/internal/platform/config" +) + +func retryPolicy(cfg config.Jobs) pipeline.RetryPolicy { + return pipeline.RetryPolicy{MaxAttempts: cfg.MaxAttempts, BaseDelay: time.Second, MaxDelay: time.Minute} +} + +func pause(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) { + for ctx.Err() == nil { + err := dispatchSession(ctx, store, cfg, barrier) + if ctx.Err() == nil { + log.Warn("attachment dispatcher will retry", "error", err) + pause(ctx, time.Second) + } + } +} + +func dispatchSession(ctx context.Context, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) error { + var publisher *broker.Publisher + publish := func(ctx context.Context, j pipeline.Job) error { + body, err := json.Marshal(j) + if err != nil { + return err + } + if err := publisher.Publish(ctx, body); err != nil { + return err + } + if barrier != nil { + return barrier(ctx, j, "published") + } + return nil + } + for ctx.Err() == nil { + if _, err := store.RecoverOne(ctx, retryPolicy(cfg)); err != nil { + return err + } + if publisher == nil { + var err error + publisher, err = broker.NewPublisher(ctx, cfg.URL) + if err != nil { + return err + } + defer publisher.Close() + } + sent, err := store.PublishOne(ctx, publish) + if err != nil { + return err + } + if !sent { + pause(ctx, 250*time.Millisecond) + } + } + return ctx.Err() +} diff --git a/internal/processing/worker.go b/internal/processing/worker.go new file mode 100644 index 00000000..1574bc0b --- /dev/null +++ b/internal/processing/worker.go @@ -0,0 +1,215 @@ +package processing + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + "os/exec" + "sync" + "time" + + "github.com/minio/minio-go/v7" + + "messenger/internal/attachment" + "messenger/internal/broker" + "messenger/internal/imaging" + "messenger/internal/pipeline" + "messenger/internal/platform/config" +) + +var errSourceMismatch = errors.New("original differs from accepted metadata") + +type Worker struct { + Store *pipeline.Store + Blobs attachment.Blobs + Config config.Jobs + Log *slog.Logger + // An experiment may pause an individual attempt at an exact durability boundary. + Checkpoint func(context.Context, pipeline.Job, string) error +} + +func (w *Worker) Run(ctx context.Context) { + var wg sync.WaitGroup + for range w.Config.Concurrency { + wg.Go(func() { + for ctx.Err() == nil { + err := w.consume(ctx) + if ctx.Err() == nil { + w.Log.Warn("consumer reconnecting", "error", err) + pause(ctx, time.Second) + } + } + }) + } + wg.Wait() +} + +func (w *Worker) consume(ctx context.Context) error { + c, err := broker.NewConsumer(ctx, w.Config.URL) + if err != nil { + return err + } + defer c.Close() + for d := range c.Deliveries { + var j pipeline.Job + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF { + w.Log.Warn("discarding invalid attachment envelope") + if err := d.Reject(false); err != nil { + return err + } + continue + } + if err := w.process(ctx, j); err != nil { + return err + } + if err := d.Ack(false); err != nil { + return err + } + if err := w.checkpoint(ctx, j, "acked"); err != nil { + return err + } + } + return errors.New("broker delivery stream closed") +} + +func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error { + if w.Checkpoint != nil { + return w.Checkpoint(ctx, j, phase) + } + return nil +} + +func (w *Worker) process(ctx context.Context, j pipeline.Job) error { + c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second) + if err != nil { + return err + } + if c == nil { + return w.checkpoint(ctx, j, "duplicate") + } + if err := w.checkpoint(ctx, j, "claimed"); err != nil { + return err + } + jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout) + defer cancel() + variants, failure := w.writeObjects(jobCtx, c) + if failure != nil { + if ctx.Err() != nil { + return ctx.Err() + } + _, err = w.Store.Fail(ctx, c, *failure, retryPolicy(w.Config)) + return err + } + if err := w.checkpoint(ctx, j, "objects"); err != nil { + return err + } + saved, err := w.Store.Complete(ctx, c, variants) + if err != nil || !saved { + return err + } + return w.checkpoint(ctx, j, "committed") +} + +func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) { + if c.SizeBytes < 1 || c.SizeBytes > imaging.MaxUploadBytes { + return nil, errSourceMismatch + } + r, err := blobs.Open(ctx, c.SourceKey) + if err != nil { + return nil, err + } + data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1)) + closeErr := r.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + h := sha256.Sum256(data) + if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) { + return nil, errSourceMismatch + } + return data, nil +} + +// Log only classified reasons. Storage and codec errors may contain URLs, +// object keys, filenames or input-derived diagnostics. +func (w *Worker) attemptFailure(c *pipeline.Claim, stage string, err error) *pipeline.Failure { + failure := &pipeline.Failure{Code: "processing_failed"} + reason := "operation_failed" + var network net.Error + var exit *exec.ExitError + var storage minio.ErrorResponse + switch { + case errors.Is(err, imaging.ErrInvalidImage): + failure.Code, failure.Permanent = "invalid_image", true + reason = failure.Code + case errors.Is(err, errSourceMismatch): + failure.Code, failure.Permanent = "source_mismatch", true + reason = failure.Code + case errors.Is(err, attachment.ErrObjectMissing): + failure.Code = "source_missing" + reason = failure.Code + case errors.Is(err, context.DeadlineExceeded): + reason = "deadline_exceeded" + case errors.Is(err, context.Canceled): + reason = "canceled" + case errors.As(err, &exit): + reason = fmt.Sprintf("codec_exit_%d", exit.ExitCode()) + case errors.Is(err, exec.ErrNotFound): + reason = "codec_not_found" + case errors.As(err, &storage): + reason = fmt.Sprintf("s3_http_%d", storage.StatusCode) + case errors.As(err, &network): + reason = "network_error" + if network.Timeout() { + reason = "network_timeout" + } + case errors.Is(err, os.ErrPermission): + reason = "permission_denied" + } + w.Log.Warn("attachment attempt failed", "job_id", c.JobID, "attempt", c.Attempt, + "error_code", failure.Code, "stage", stage, "reason", reason) + return failure +} + +func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, *pipeline.Failure) { + data, err := readOriginal(ctx, w.Blobs, c) + if err != nil { + return nil, w.attemptFailure(c, "read_original", err) + } + encoded, err := imaging.Render(ctx, data, c.PipelineVersion) + if err != nil { + return nil, w.attemptFailure(c, "render", err) + } + key := attachment.BlobKey(c.Hash) + if c.SourceKey != key { + if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil { + return nil, w.attemptFailure(c, "write_original", err) + } + } + var variants []pipeline.Variant + for _, v := range encoded { + hash := sha256.Sum256(v.Data) + key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:]) + n := int64(len(v.Data)) + if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil { + return nil, w.attemptFailure(c, "write_variant", err) + } + variants = append(variants, pipeline.Variant{ + Name: v.Name, Key: key, MIMEType: v.MIMEType, + Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:], + }) + } + return variants, nil +} diff --git a/internal/processing/worker_test.go b/internal/processing/worker_test.go new file mode 100644 index 00000000..45d2bce4 --- /dev/null +++ b/internal/processing/worker_test.go @@ -0,0 +1,64 @@ +package processing + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "testing" + + "messenger/internal/attachment" + "messenger/internal/pipeline" +) + +type stagedSource struct { + attachment.Blobs + body string +} + +func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(s.body)), nil +} + +func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) { + h := sha256.Sum256([]byte("accepted")) + c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"} + for _, bad := range []string{"accept", "altered!", "oversized bytes"} { + _, err := readOriginal(t.Context(), stagedSource{body: bad}, c) + if !errors.Is(err, errSourceMismatch) { + t.Fatalf("body %q: %v", bad, err) + } + } + got, err := readOriginal(t.Context(), stagedSource{body: "accepted"}, c) + if err != nil || string(got) != "accepted" { + t.Fatalf("valid original: %v", err) + } +} + +type failingSource struct{ attachment.Blobs } + +func (failingSource) Open(context.Context, string) (io.ReadCloser, error) { + return nil, fmt.Errorf("credentials=private-source-data: %w", context.DeadlineExceeded) +} + +func TestFailureLogReportsStageAndReason(t *testing.T) { + var log strings.Builder + w := Worker{Blobs: failingSource{}, Log: slog.New(slog.NewJSONHandler(&log, nil))} + c := &pipeline.Claim{SizeBytes: 5, SourceKey: "sensitive-source-key"} + _, failure := w.writeObjects(t.Context(), c) + if failure == nil { + t.Fatal("expected failure reading source") + } + got := log.String() + for _, want := range []string{`"stage":"read_original"`, `"reason":"deadline_exceeded"`, `"error_code":"processing_failed"`, `"job_id":`, `"attempt":`} { + if !strings.Contains(got, want) { + t.Errorf("log missing %s: %s", want, got) + } + } + if strings.Contains(got, "private-source-data") || strings.Contains(got, "sensitive-source-key") { + t.Errorf("leaked source details: %s", got) + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 00000000..661c5e10 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,83 @@ +// Package server runs the HTTP listener with a graceful stop behind a load +// balancer. +package server + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "time" +) + +// Readiness is switched off when shutdown starts. +type Readiness interface { + SetReady(ready bool) +} + +// Run serves until ctx is cancelled, keeps serving for drainDelay with +// readiness off, then shuts down within shutdownTimeout. The balancer learns +// of the stop only from failing readiness checks, so drainDelay must exceed +// its detection time. +func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, ready Readiness, drainDelay, shutdownTimeout time.Duration) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 35 * time.Second, + WriteTimeout: 35 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 64 * 1024, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + + served := make(chan error, 1) + go func() { + err := srv.Serve(ln) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + served <- err + }() + log.Info("listening", "addr", ln.Addr().String()) + + select { + case err := <-served: + return err + case <-ctx.Done(): + } + + log.Info("shutdown started", "drain_delay", drainDelay.String(), "timeout", shutdownTimeout.String()) + ready.SetReady(false) + time.Sleep(drainDelay) + drain, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := srv.Shutdown(drain); err != nil { + log.Error("shutdown forced", "error", err.Error()) + _ = srv.Close() + return err + } + <-served + log.Info("shutdown completed") + return nil +} + +// RunDebug serves the internal metrics until ctx is cancelled; no request +// depends on them, so it stops without draining. +func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + go func() { + <-ctx.Done() + _ = srv.Close() + }() + log.Info("debug listening", "addr", ln.Addr().String()) + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 00000000..c8685c72 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,346 @@ +package server_test + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/gin-gonic/gin" + + "messenger/internal/health" + "messenger/internal/platform/config" + "messenger/internal/platform/httpx" + "messenger/internal/server" +) + +func serverDefaultConfig(t *testing.T) config.Config { + t.Helper() + values := make(map[string]string) + for key, content := range map[string]string{ + "DATABASE_URL_FILE": "postgres://unused:unused@localhost:5432/unused", + "SESSION_STORE_URL_FILE": "redis://localhost:6379/0", + } { + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + values[key] = path + } + cfg, err := config.Load(func(key string) (string, bool) { value, ok := values[key]; return value, ok }) + if err != nil { + t.Fatal(err) + } + return cfg +} + +func TestDefaultShutdownAllowsActiveBinaryTransfers(t *testing.T) { + for _, direction := range []string{"upload", "download"} { + t.Run(direction, func(t *testing.T) { + t.Parallel() + cfg := serverDefaultConfig(t) + payload := []byte("binary\x00\xff\x01request-and-response\x00\xfe") + prefix := payload[:3] + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + bodyReader, bodyWriter := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + requestCtx, cancelRequest := context.WithTimeout(context.Background(), 25*time.Second) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + probes := health.New() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + if direction == "upload" { + first := make([]byte, len(prefix)) + if _, err := io.ReadFull(r.Body, first); err != nil { + return + } + close(started) + rest, err := io.ReadAll(r.Body) + if err != nil || !bytes.Equal(append(first, rest...), payload) { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = w.Write(payload) + return + } + _, _ = w.Write(prefix) + w.(http.Flusher).Flush() + close(started) + select { + case <-release: + _, _ = w.Write(payload[len(prefix):]) + case <-r.Context().Done(): + } + }) + stopped, exited := make(chan error, 1), make(chan struct{}) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + go func() { + defer close(exited) + stopped <- server.Run(ctx, log, ln, handler, probes, cfg.DrainDelay, cfg.ShutdownTimeout) + }() + t.Cleanup(func() { + cancel() + cancelRequest() + unblock() + _ = bodyReader.Close() + _ = bodyWriter.Close() + _ = ln.Close() + select { + case <-exited: + case <-time.After(3 * time.Second): + t.Error("server did not stop after transfer cleanup") + } + }) + client := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + defer client.CloseIdleConnections() + method := http.MethodGet + var body io.Reader + if direction == "upload" { + method = http.MethodPut + body = bodyReader + } + req, err := http.NewRequestWithContext(requestCtx, method, "http://"+ln.Addr().String()+"/binary", body) + if err != nil { + t.Fatal(err) + } + if direction == "upload" { + req.ContentLength = int64(len(payload)) + req.Header.Set("Content-Type", "application/octet-stream") + } + completed := make(chan error, 1) + go func() { + resp, err := client.Do(req) + if err != nil { + completed <- err + return + } + defer resp.Body.Close() + got, err := io.ReadAll(resp.Body) + if err == nil && (resp.StatusCode != http.StatusOK || !bytes.Equal(got, payload)) { + err = fmt.Errorf("binary response status=%d body=%q, want exact payload", resp.StatusCode, got) + } + completed <- err + }() + if direction == "upload" { + if _, err := bodyWriter.Write(prefix); err != nil { + t.Fatal(err) + } + } + select { + case <-started: + case err := <-completed: + t.Fatalf("transfer failed before shutdown: %v", err) + case <-requestCtx.Done(): + t.Fatal("transfer never became active") + } + cancel() + // Nine seconds inside Shutdown exceeds the former 8s budget. Include + // the configured drain phase so it cannot hide a premature force-close. + timer := time.NewTimer(cfg.DrainDelay + 9*time.Second) + defer timer.Stop() + select { + case err := <-stopped: + t.Fatalf("shutdown interrupted active %s before binary transfer completed: %v", direction, err) + case err := <-completed: + t.Fatalf("active %s ended before remaining bytes were released: %v", direction, err) + case <-timer.C: + } + if direction == "upload" { + if _, err := bodyWriter.Write(payload[len(prefix):]); err != nil { + t.Fatal(err) + } + if err := bodyWriter.Close(); err != nil { + t.Fatal(err) + } + } else { + unblock() + } + select { + case err := <-completed: + if err != nil { + t.Fatalf("active %s was truncated during shutdown: %v", direction, err) + } + case <-requestCtx.Done(): + t.Fatal("binary transfer did not finish after release") + } + select { + case err := <-stopped: + if err != nil { + t.Fatalf("graceful shutdown: %v", err) + } + case <-requestCtx.Done(): + t.Fatal("server did not finish graceful shutdown") + } + if probes.IsReady() { + t.Fatal("readiness remained enabled after shutdown") + } + }) + } +} + +// newHandler builds the smallest handler with probes; the lifecycle does not +// depend on the business modules or their stores. +func newHandler(log *slog.Logger) (*gin.Engine, *health.Handler) { + e := httpx.NewEngine(log, 1024, "test") + probes := health.New() + probes.Register(e) + return e, probes +} + +func TestServesUntilContextCancelledThenStopsCleanly(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() + + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop within 10s after cancel") + } + if _, err := http.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after shutdown") + } +} + +func TestReadinessDropsBeforeListenerCloses(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.Run(ctx, log, ln, h, probes, 0, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + // Hold one request open so shutdown has something to drain. + slow := make(chan struct{}) + go func() { + resp, err := http.Get(base + "/health/live") + if err == nil { + resp.Body.Close() + } + close(slow) + }() + + cancel() + <-done + <-slow + if !probes.IsReady() { + return + } + t.Fatal("readiness must be switched off during shutdown") +} + +// During the drain delay the balancer is still learning that the replica is +// going away: readiness already fails, yet new connections are accepted and +// served. Only after the delay does the listener close. +func TestDrainKeepsServingWhileNotReady(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + h, probes := newHandler(log) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + const drain = 700 * time.Millisecond + go func() { done <- server.Run(ctx, log, ln, h, probes, drain, 5*time.Second) }() + base := "http://" + ln.Addr().String() + waitFor(t, base+"/health/ready", http.StatusOK) + + cancel() + waitFor(t, base+"/health/ready", http.StatusServiceUnavailable) + // A fresh connection, as the balancer opens for a new request. + fresh := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + resp, err := fresh.Get(base + "/health/live") + if err != nil { + t.Fatalf("new connection during the drain delay: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("liveness during drain = %d, want 200", resp.StatusCode) + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop after the drain delay") + } + if _, err := fresh.Get(base + "/health/live"); err == nil { + t.Fatal("listener must be closed after the drain delay") + } +} + +func waitFor(t *testing.T, url string, status int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + resp, err := http.Get(url) + if err == nil { + resp.Body.Close() + if resp.StatusCode == status { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not return %d in time", url, status) +} + +// The debug listener has no drain: it serves the benchmark's metrics scraper +// only and stops with the process. +func TestRunDebugServesUntilCancelled(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.RunDebug(ctx, log, ln, h) }() + waitFor(t, "http://"+ln.Addr().String()+"/debug/vars", http.StatusTeapot) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunDebug returned %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunDebug did not stop after cancel") + } +} diff --git a/internal/testenv/s3.go b/internal/testenv/s3.go new file mode 100644 index 00000000..82fb0a44 --- /dev/null +++ b/internal/testenv/s3.go @@ -0,0 +1,207 @@ +package testenv + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/google/uuid" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "messenger/internal/platform/config" + "messenger/internal/platform/s3store" +) + +const S3Image = "chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0" + +// S3 keeps independent API, reconciler and bootstrap identities. Store uses the +// reconciler identity so tests can inspect and clean up their objects. +type S3 struct { + Store, APIStore, Admin *s3store.Store + Endpoint, Region, Bucket string + GRPCAddress string + AccessKey, SecretKey string + APIAccessKey, APISecretKey string + AdminAccessKey, AdminSecretKey string + container testcontainers.Container + volume string + identities, options []byte +} + +func StartS3(ctx context.Context) (*S3, error) { + useDockerDesktopPipe() + s := &S3{Region: "us-east-1", Bucket: "messenger-attachments", volume: "lab5-s3-test-" + uuid.NewString()} + keys := []*string{&s.AccessKey, &s.SecretKey, &s.APIAccessKey, &s.APISecretKey, &s.AdminAccessKey, &s.AdminSecretKey} + for _, key := range keys { + b := make([]byte, 24) + if _, err := rand.Read(b); err != nil { + return nil, err + } + *key = hex.EncodeToString(b) + } + type credential struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + } + type identity struct { + Name string `json:"name"` + Credentials []credential `json:"credentials"` + Actions []string `json:"actions"` + } + config := struct { + Identities []identity `json:"identities"` + }{Identities: []identity{ + {"bootstrap", []credential{{s.AdminAccessKey, s.AdminSecretKey}}, []string{"Admin", "Read", "Write", "List"}}, + {"api", []credential{{s.APIAccessKey, s.APISecretKey}}, []string{"Read:" + s.Bucket, "Write:" + s.Bucket}}, + {"reconciler", []credential{{s.AccessKey, s.SecretKey}}, []string{"List:" + s.Bucket, "Read:" + s.Bucket, "Write:" + s.Bucket}}, + }} + var err error + s.identities, err = json.Marshal(config) + if err != nil { + return nil, err + } + _, source, _, _ := runtime.Caller(0) + s.options, err = os.ReadFile(filepath.Join(filepath.Dir(source), "../../infra/s3/seaweedfs.options")) + if err != nil { + return nil, fmt.Errorf("read S3 options: %w", err) + } + startCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + if err := s.launch(startCtx); err != nil { + s.Close() + return nil, err + } + if err := s.Admin.Bootstrap(startCtx); err != nil { + s.Close() + return nil, err + } + return s, nil +} + +func (s *S3) launch(ctx context.Context) error { + pids := int64(256) + firstStatus := 0 + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: S3Image, + Cmd: []string{"server", "-options=/etc/seaweedfs/lab.options"}, + Env: map[string]string{"GOMEMLIMIT": "460MiB"}, + ExposedPorts: []string{"8333/tcp", "18333/tcp"}, + Mounts: testcontainers.Mounts(testcontainers.VolumeMount(s.volume, "/data")), + Files: []testcontainers.ContainerFile{ + {Reader: bytes.NewReader(s.identities), ContainerFilePath: "/run/secrets/s3_identities", FileMode: 0444}, + {Reader: bytes.NewReader(s.options), ContainerFilePath: "/etc/seaweedfs/lab.options", FileMode: 0444}, + }, + HostConfigModifier: func(h *container.HostConfig) { + h.Resources.Memory = 512 * 1024 * 1024 + h.Resources.NanoCPUs = 1_000_000_000 + h.Resources.PidsLimit = &pids + h.PortBindings = nat.PortMap{"8333/tcp": {{HostIP: "127.0.0.1"}}, "18333/tcp": {{HostIP: "127.0.0.1"}}} + }, + // The first reachable S3 response must already reject anonymous access. + WaitingFor: wait.ForHTTP("/").WithPort("8333/tcp").WithStatusCodeMatcher(func(code int) bool { firstStatus = code; return true }).WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + s.container = c + if err != nil { + return fmt.Errorf("start S3: %w", err) + } + if firstStatus != http.StatusForbidden { + return fmt.Errorf("first anonymous S3 response was %d, expected 403", firstStatus) + } + s.Endpoint, err = c.PortEndpoint(ctx, "8333/tcp", "http") + if err != nil { + return err + } + s.GRPCAddress, err = c.PortEndpoint(ctx, "18333/tcp", "") + if err != nil { + return err + } + s.Store, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AccessKey, SecretKey: s.SecretKey}) + if err != nil { + return err + } + s.APIStore, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.APIAccessKey, SecretKey: s.APISecretKey}) + if err != nil { + return err + } + s.Admin, err = s3store.New(config.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, AccessKey: s.AdminAccessKey, SecretKey: s.AdminSecretKey}) + return err +} + +func (s *S3) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return s.container.Stop(ctx, &timeout) +} + +func (s *S3) Start(ctx context.Context) error { + if err := s.container.Start(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Recreate replaces the container while keeping only its named /data volume. +// Bootstrap is deliberately not called: bucket metadata must survive as well. +func (s *S3) Recreate(ctx context.Context) error { + if err := testcontainers.TerminateContainer(s.container); err != nil { + return err + } + s.container = nil + if err := s.launch(ctx); err != nil { + return err + } + return s.Admin.WaitReady(ctx) +} + +// Diagnostics retains bounded startup evidence without exposing fixture keys. +func (s *S3) Diagnostics(ctx context.Context) string { + if s.container == nil { + return "no container" + } + r, err := s.container.Logs(ctx) + if err != nil { + return err.Error() + } + defer r.Close() + redact := strings.NewReplacer(s.AccessKey, "", s.SecretKey, "", s.APIAccessKey, "", s.APISecretKey, "", s.AdminAccessKey, "", s.AdminSecretKey, "") + var lines []string + scan := bufio.NewScanner(io.LimitReader(r, 256*1024)) + for scan.Scan() { + line := scan.Text() + lower := strings.ToLower(line) + if strings.Contains(lower, "volume") || strings.Contains(lower, "master") || strings.Contains(lower, "leader") || strings.Contains(lower, "error") || strings.Contains(lower, "lookup") { + lines = append(lines, redact.Replace(line)) + if len(lines) > 60 { + lines = lines[1:] + } + } + } + return strings.Join(lines, "\n") +} + +func (s *S3) Close() { + if s.container != nil { + _ = testcontainers.TerminateContainer(s.container) + } + if strings.HasPrefix(s.volume, "lab5-s3-test-") { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = withDocker(ctx, func(cli *testcontainers.DockerClient) error { return cli.VolumeRemove(ctx, s.volume, true) }) + } +} diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go new file mode 100644 index 00000000..57cf53bd --- /dev/null +++ b/internal/testenv/testenv.go @@ -0,0 +1,181 @@ +// Package testenv starts disposable PostgreSQL and Valkey containers for tests +// that exercise the real stores. Only _test files import it, so it never +// reaches the server binary. The images match compose.yaml. +package testenv + +import ( + "context" + "fmt" + "os" + "runtime" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcvalkey "github.com/testcontainers/testcontainers-go/modules/valkey" + + "messenger/internal/platform/postgres" + "messenger/internal/platform/valkey" +) + +const ( + PostgresImage = "postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873" + ValkeyImage = "valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11" +) + +// useDockerDesktopPipe names the Docker Desktop engine pipe explicitly on +// Windows. Without DOCKER_HOST, testcontainers probes the pipe with a file +// stat, which fails while the pipe is busy; with several test packages +// starting containers at once that probe fails spuriously and the run panics. +// Dialling the named host instead waits for a free pipe instance. +var useDockerDesktopPipe = sync.OnceFunc(func() { + if runtime.GOOS == "windows" && os.Getenv("DOCKER_HOST") == "" { + _ = os.Setenv("DOCKER_HOST", "npipe:////./pipe/docker_engine") + } +}) + +// DB is a migrated PostgreSQL instance. +type DB struct { + Pool *pgxpool.Pool + DSN string + container *tcpostgres.PostgresContainer +} + +func StartPostgres(ctx context.Context) (*DB, error) { + useDockerDesktopPipe() + c, err := tcpostgres.Run(ctx, PostgresImage, + tcpostgres.WithDatabase("messenger"), + tcpostgres.WithUsername("messenger"), + tcpostgres.WithPassword("messenger"), + tcpostgres.BasicWaitStrategies(), + ) + if err != nil { + return nil, fmt.Errorf("start postgres: %w", err) + } + db := &DB{container: c} + db.DSN, err = c.ConnectionString(ctx, "sslmode=disable") + if err != nil { + db.Close() + return nil, fmt.Errorf("postgres address: %w", err) + } + db.Pool, err = postgres.NewPool(ctx, db.DSN, 10) + if err != nil { + db.Close() + return nil, err + } + if _, err := postgres.Migrate(ctx, db.Pool); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// Reset empties every application table, so each test starts from an empty +// schema. The table list comes from the catalog: a table added by a later +// migration cannot be forgotten here. +func (d *DB) Reset(t testing.TB) { + t.Helper() + ctx := context.Background() + var tables string + err := d.Pool.QueryRow(ctx, ` + SELECT coalesce(string_agg(format('%I.%I', schemaname, tablename), ', '), '') + FROM pg_tables + WHERE schemaname = 'public' AND tablename <> 'goose_db_version'`).Scan(&tables) + if err != nil { + t.Fatalf("list tables: %v", err) + } + if tables == "" { + return + } + if _, err := d.Pool.Exec(ctx, "TRUNCATE "+tables+" CASCADE"); err != nil { + t.Fatalf("truncate: %v", err) + } +} + +// Stop stops the container, as when the database goes down. +func (d *DB) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return d.container.Stop(ctx, &timeout) +} + +// Pause freezes the server: connections stay open but nothing answers, as +// with a hung database. +func (d *DB) Pause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerPause(ctx, d.container.GetContainerID()) + }) +} + +func (d *DB) Unpause(ctx context.Context) error { + return withDocker(ctx, func(cli *testcontainers.DockerClient) error { + return cli.ContainerUnpause(ctx, d.container.GetContainerID()) + }) +} + +func withDocker(ctx context.Context, fn func(cli *testcontainers.DockerClient) error) error { + cli, err := testcontainers.NewDockerClientWithOpts(ctx) + if err != nil { + return err + } + defer cli.Close() + return fn(cli) +} + +func (d *DB) Close() { + if d.Pool != nil { + d.Pool.Close() + } + _ = testcontainers.TerminateContainer(d.container) +} + +// KV is a Valkey instance without persistence or password. +type KV struct { + Client *redis.Client + URL string + container *tcvalkey.ValkeyContainer +} + +func StartValkey(ctx context.Context) (*KV, error) { + useDockerDesktopPipe() + c, err := tcvalkey.Run(ctx, ValkeyImage) + if err != nil { + return nil, fmt.Errorf("start valkey: %w", err) + } + kv := &KV{container: c} + kv.URL, err = c.ConnectionString(ctx) + if err != nil { + kv.Close() + return nil, fmt.Errorf("valkey address: %w", err) + } + // The production client, so outage tests see its fail-fast settings. + kv.Client, err = valkey.NewClient(ctx, kv.URL) + if err != nil { + kv.Close() + return nil, err + } + return kv, nil +} + +func (k *KV) Reset(t testing.TB) { + t.Helper() + if err := k.Client.FlushDB(context.Background()).Err(); err != nil { + t.Fatalf("flushdb: %v", err) + } +} + +// Stop stops the container, as when the session store goes down. +func (k *KV) Stop(ctx context.Context) error { + timeout := 5 * time.Second + return k.container.Stop(ctx, &timeout) +} + +func (k *KV) Close() { + if k.Client != nil { + _ = k.Client.Close() + } + _ = testcontainers.TerminateContainer(k.container) +} diff --git a/internal/testenv/testenv_test.go b/internal/testenv/testenv_test.go new file mode 100644 index 00000000..6488bded --- /dev/null +++ b/internal/testenv/testenv_test.go @@ -0,0 +1,29 @@ +package testenv_test + +import ( + "context" + "testing" + + "messenger/internal/testenv" +) + +func TestValkeyStartsAndResetFlushesKeys(t *testing.T) { + ctx := context.Background() + kv, err := testenv.StartValkey(ctx) + if err != nil { + t.Fatal(err) + } + defer kv.Close() + + if err := kv.Client.Set(ctx, "probe", "1", 0).Err(); err != nil { + t.Fatal(err) + } + kv.Reset(t) + n, err := kv.Client.DBSize(ctx).Result() + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("%d keys left after Reset", n) + } +} diff --git a/internal/user/handler.go b/internal/user/handler.go new file mode 100644 index 00000000..a5f091af --- /dev/null +++ b/internal/user/handler.go @@ -0,0 +1,70 @@ +package user + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "messenger/internal/platform/httpx" +) + +type Handler struct { + users *Service +} + +func NewHandler(users *Service) *Handler { + return &Handler{users: users} +} + +// Register mounts registration on the public router and the current-user +// endpoint on the router guarded by the authentication middleware. +func (h *Handler) Register(public, protected gin.IRouter) { + public.POST("/users", h.register) + protected.GET("/users/me", httpx.WithUser(h.me)) +} + +type createRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type Response struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +func ToResponse(u User) Response { + return Response{ + ID: u.ID.String(), + Username: u.Username, + DisplayName: u.DisplayName, + CreatedAt: httpx.Timestamp(u.CreatedAt), + } +} + +func (h *Handler) register(c *gin.Context) { + var req createRequest + if err := httpx.DecodeJSON(c, &req); err != nil { + httpx.Error(c, err) + return + } + u, err := h.users.Register(c.Request.Context(), req.Username, req.DisplayName, req.Password) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusCreated, ToResponse(u)) +} + +func (h *Handler) me(c *gin.Context, actor uuid.UUID) { + u, err := h.users.GetByID(c.Request.Context(), actor) + if err != nil { + httpx.Error(c, err) + return + } + httpx.JSON(c, http.StatusOK, ToResponse(u)) +} diff --git a/internal/user/main_test.go b/internal/user/main_test.go new file mode 100644 index 00000000..ba3a1ca2 --- /dev/null +++ b/internal/user/main_test.go @@ -0,0 +1,24 @@ +package user_test + +import ( + "context" + "fmt" + "os" + "testing" + + "messenger/internal/testenv" +) + +var db *testenv.DB + +func TestMain(m *testing.M) { + var err error + db, err = testenv.StartPostgres(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + db.Close() + os.Exit(code) +} diff --git a/internal/user/repository.go b/internal/user/repository.go new file mode 100644 index 00000000..eb2ac2f6 --- /dev/null +++ b/internal/user/repository.go @@ -0,0 +1,18 @@ +package user + +import ( + "context" + + "github.com/google/uuid" +) + +type Repository interface { + // Create stores a new user or returns ErrUsernameTaken; the uniqueness + // check and the insert must be atomic. + Create(ctx context.Context, user User) error + GetByID(ctx context.Context, id uuid.UUID) (User, error) + GetByUsername(ctx context.Context, username string) (User, error) + // ExistingIDs returns the subset of ids that belong to registered users, + // in no particular order, in one round trip. + ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) +} diff --git a/internal/user/service.go b/internal/user/service.go new file mode 100644 index 00000000..708ddb9b --- /dev/null +++ b/internal/user/service.go @@ -0,0 +1,88 @@ +package user + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/google/uuid" + + "messenger/internal/platform/ident" + "messenger/internal/platform/password" +) + +type Service struct { + repo Repository +} + +func NewService(repo Repository) *Service { + return &Service{repo: repo} +} + +// Register validates every field before hashing, so invalid input never pays +// for the KDF. +func (s *Service) Register(ctx context.Context, username, displayName, plain string) (User, error) { + u, err := New(ident.New(), username, displayName) + if err != nil { + return User{}, err + } + if err := ValidatePassword(plain); err != nil { + return User{}, err + } + u.PasswordHash = password.Hash(plain) + if err := s.repo.Create(ctx, u); err != nil { + return User{}, err + } + return u, nil +} + +// dummyHash is verified for unknown usernames, so a failed login takes the +// same time whether or not the user exists. +var dummyHash = sync.OnceValue(func() string { + return password.Hash("timing-equaliser-password") +}) + +// Authenticate returns the user whose password matches, or +// ErrInvalidCredentials for an unknown username and a wrong password alike. +// The username comes straight from the Basic header, so one that cannot +// exist (including NUL or invalid UTF-8, which the database would reject) +// is answered without a query, but after the same KDF work. +func (s *Service) Authenticate(ctx context.Context, username, plain string) (User, error) { + if !usernamePattern.MatchString(username) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + u, err := s.repo.GetByUsername(ctx, username) + if errors.Is(err, ErrNotFound) { + _, _ = password.Verify(dummyHash(), plain) + return User{}, ErrInvalidCredentials + } + if err != nil { + return User{}, fmt.Errorf("find user: %w", err) + } + ok, err := password.Verify(u.PasswordHash, plain) + if err != nil { + return User{}, fmt.Errorf("verify password of user %s: %w", u.ID, err) + } + if !ok { + return User{}, ErrInvalidCredentials + } + return u, nil +} + +func (s *Service) GetByID(ctx context.Context, id uuid.UUID) (User, error) { + return s.repo.GetByID(ctx, id) +} + +// ExistingIDs serves the chats module when it validates member lists. +func (s *Service) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + if len(ids) == 0 { + return []uuid.UUID{}, nil + } + found, err := s.repo.ExistingIDs(ctx, ids) + if err != nil { + return nil, fmt.Errorf("user lookup: %w", err) + } + return found, nil +} diff --git a/internal/user/service_test.go b/internal/user/service_test.go new file mode 100644 index 00000000..3406ddd8 --- /dev/null +++ b/internal/user/service_test.go @@ -0,0 +1,189 @@ +package user_test + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" + "messenger/internal/user/store" +) + +const pw = "correct-horse-battery" + +// newService wires the service to the real store over an empty database. +func newService(t *testing.T) *user.Service { + t.Helper() + db.Reset(t) + return user.NewService(store.NewUsers(db.Pool)) +} + +func TestRegisterStoresUserWithPasswordHash(t *testing.T) { + svc := newService(t) + ctx := context.Background() + u, err := svc.Register(ctx, "alice", "Алиса", pw) + if err != nil { + t.Fatalf("Register error: %v", err) + } + if u.ID.Version() != 7 { + t.Fatalf("id must be UUIDv7, got version %d", u.ID.Version()) + } + if !strings.HasPrefix(u.PasswordHash, "$argon2id$") || strings.Contains(u.PasswordHash, pw) { + t.Fatalf("password must be stored as an Argon2id hash, got %q", u.PasswordHash) + } + got, err := svc.GetByID(ctx, u.ID) + if err != nil { + t.Fatalf("GetByID error: %v", err) + } + if got != u { + t.Fatalf("GetByID = %+v, want %+v", got, u) + } +} + +func TestRegisterRejectsInvalidInputAndStoresNothing(t *testing.T) { + svc := newService(t) + ctx := context.Background() + cases := map[string][3]string{ + "username": {"Bad Name", "x", pw}, + "display_name": {"alice", " ", pw}, + "password": {"alice", "Алиса", "short"}, + } + for field, in := range cases { + _, err := svc.Register(ctx, in[0], in[1], in[2]) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), field) { + t.Errorf("%s: err = %v, want KindInvalid naming the field", field, err) + } + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&n); err != nil || n != 0 { + t.Fatalf("users = %d, %v; invalid input must not be stored", n, err) + } +} + +func TestRegisterReportsDuplicateAsConflict(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "A", pw); err != nil { + t.Fatal(err) + } + _, err := svc.Register(ctx, "alice", "B", pw) + if !errors.Is(err, user.ErrUsernameTaken) || apperr.KindOf(err) != apperr.KindConflict { + t.Fatalf("err = %v, want ErrUsernameTaken with KindConflict", err) + } +} + +// Concurrent registrations of one username race on the unique constraint: +// exactly one wins and every other gets the conflict, never a 500. +func TestConcurrentRegistrationOfOneUsernameHasOneWinner(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const attempts = 10 + errs := make(chan error, attempts) + var start, done sync.WaitGroup + start.Add(1) + for range attempts { + done.Add(1) + go func() { + defer done.Done() + start.Wait() + _, err := svc.Register(ctx, "racer", "Racer", pw) + errs <- err + }() + } + start.Done() + done.Wait() + close(errs) + + won, lost := 0, 0 + for err := range errs { + switch { + case err == nil: + won++ + case errors.Is(err, user.ErrUsernameTaken): + lost++ + default: + t.Errorf("unexpected error: %v", err) + } + } + if won != 1 || lost != attempts-1 { + t.Fatalf("won = %d, lost = %d; want 1 and %d", won, lost, attempts-1) + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM users WHERE username = 'racer'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("rows = %d, %v; want exactly one", n, err) + } +} + +func TestGetByIDUnknownIsNotFound(t *testing.T) { + svc := newService(t) + _, err := svc.GetByID(context.Background(), ident.New()) + if !errors.Is(err, user.ErrNotFound) || apperr.KindOf(err) != apperr.KindNotFound { + t.Fatalf("err = %v, want ErrNotFound with KindNotFound", err) + } +} + +func TestExistingIDsReturnsOnlyRegistered(t *testing.T) { + svc := newService(t) + ctx := context.Background() + alice, _ := svc.Register(ctx, "alice", "A", pw) + bob, _ := svc.Register(ctx, "bob", "B", pw) + got, err := svc.ExistingIDs(ctx, []uuid.UUID{ident.New(), bob.ID, alice.ID}) + if err != nil { + t.Fatal(err) + } + set := map[uuid.UUID]bool{} + for _, id := range got { + set[id] = true + } + if len(got) != 2 || !set[alice.ID] || !set[bob.ID] { + t.Fatalf("ExistingIDs = %v, want alice and bob", got) + } + if empty, err := svc.ExistingIDs(ctx, nil); err != nil || len(empty) != 0 { + t.Fatalf("ExistingIDs(nil) = %v, %v", empty, err) + } +} + +func TestAuthenticateChecksPassword(t *testing.T) { + svc := newService(t) + ctx := context.Background() + const colonPassword = "пароль:с:двоеточием 👋" + alice, err := svc.Register(ctx, "alice", "Алиса", colonPassword) + if err != nil { + t.Fatal(err) + } + got, err := svc.Authenticate(ctx, "alice", colonPassword) + if err != nil || got != alice { + t.Fatalf("Authenticate = %+v, %v; want alice", got, err) + } +} + +// A wrong password and an unknown username must be indistinguishable. +func TestAuthenticateFailuresLookTheSame(t *testing.T) { + svc := newService(t) + ctx := context.Background() + if _, err := svc.Register(ctx, "alice", "Алиса", pw); err != nil { + t.Fatal(err) + } + _, wrong := svc.Authenticate(ctx, "alice", pw+"!") + _, unknown := svc.Authenticate(ctx, "nobody", pw) + // Basic credentials are client bytes: a username the database cannot + // even compare must still be a plain credential failure. + _, nul := svc.Authenticate(ctx, "a\x00b", pw) + _, notUTF8 := svc.Authenticate(ctx, "\xff\xfe", pw) + for name, err := range map[string]error{ + "wrong password": wrong, "unknown user": unknown, "nul username": nul, "invalid utf-8 username": notUTF8, + } { + if !errors.Is(err, user.ErrInvalidCredentials) || apperr.KindOf(err) != apperr.KindUnauthorized { + t.Errorf("%s: err = %v, want ErrInvalidCredentials", name, err) + } + } + if wrong.Error() != unknown.Error() { + t.Fatalf("messages differ: %q vs %q", wrong, unknown) + } +} diff --git a/internal/user/store/postgres.go b/internal/user/store/postgres.go new file mode 100644 index 00000000..d8974847 --- /dev/null +++ b/internal/user/store/postgres.go @@ -0,0 +1,74 @@ +// Package store holds the repository implementations of the users module. +// Errors pass through postgres.Classify, so an unreachable database surfaces +// as unavailability rather than as an internal error. +package store + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/platform/postgres" + "messenger/internal/user" +) + +type Users struct { + pool *pgxpool.Pool +} + +var _ user.Repository = (*Users)(nil) + +func NewUsers(pool *pgxpool.Pool) *Users { + return &Users{pool: pool} +} + +// Create relies on the unique constraint rather than a prior lookup, so two +// concurrent registrations of one username cannot both succeed. +func (r *Users) Create(ctx context.Context, u user.User) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES ($1, $2, $3, $4, $5)`, + u.ID, u.Username, u.DisplayName, u.PasswordHash, u.CreatedAt) + if postgres.IsUniqueViolation(err, "users_username_key") { + return user.ErrUsernameTaken + } + if err != nil { + return fmt.Errorf("insert user: %w", postgres.Classify(err)) + } + return nil +} + +func (r *Users) GetByID(ctx context.Context, id uuid.UUID) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1`, id) +} + +func (r *Users) GetByUsername(ctx context.Context, username string) (user.User, error) { + return r.one(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1`, username) +} + +func (r *Users) ExistingIDs(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + rows, err := r.pool.Query(ctx, `SELECT id FROM users WHERE id = ANY($1)`, ids) + if err != nil { + return nil, fmt.Errorf("select user ids: %w", postgres.Classify(err)) + } + found, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) + if err != nil { + return nil, fmt.Errorf("read user ids: %w", postgres.Classify(err)) + } + return found, nil +} + +func (r *Users) one(ctx context.Context, query string, arg any) (user.User, error) { + var u user.User + err := r.pool.QueryRow(ctx, query, arg).Scan(&u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return user.User{}, user.ErrNotFound + } + if err != nil { + return user.User{}, fmt.Errorf("select user: %w", postgres.Classify(err)) + } + return u, nil +} diff --git a/internal/user/user.go b/internal/user/user.go new file mode 100644 index 00000000..d05e1c43 --- /dev/null +++ b/internal/user/user.go @@ -0,0 +1,75 @@ +// Package user is the users module. The entity mirrors the table row and +// carries no behaviour; New only enforces input invariants and CreatedAt +// derives from the UUIDv7 id. The password itself is never kept: the service +// validates it and stores only its hash. +package user + +import ( + "fmt" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" +) + +const ( + MaxDisplayNameCodePoints = 100 + MinPasswordCodePoints = 12 + MaxPasswordCodePoints = 128 +) + +var ( + ErrNotFound = apperr.NotFound("user not found") + ErrUsernameTaken = apperr.Conflict("username already taken") + // ErrInvalidCredentials covers an unknown username and a wrong password + // alike, so a login attempt cannot probe which usernames exist. + ErrInvalidCredentials = apperr.Unauthorized("invalid username or password") +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +func New(id uuid.UUID, username, displayName string) (User, error) { + if !usernamePattern.MatchString(username) { + return User{}, apperr.Invalid("username must match [a-z0-9_]{3,32}") + } + if strings.TrimSpace(displayName) == "" { + return User{}, apperr.Invalid("display_name must not be blank") + } + if utf8.RuneCountInString(displayName) > MaxDisplayNameCodePoints { + return User{}, apperr.Invalid(fmt.Sprintf("display_name must be at most %d characters", MaxDisplayNameCodePoints)) + } + // PostgreSQL text cannot hold U+0000. + if strings.ContainsRune(displayName, 0) { + return User{}, apperr.Invalid("display_name must not contain the NUL character") + } + return User{ + ID: id, + Username: username, + DisplayName: displayName, + CreatedAt: ident.TimeOf(id), + }, nil +} + +// ValidatePassword checks the contract length in Unicode code points. +func ValidatePassword(plain string) error { + if strings.TrimSpace(plain) == "" { + return apperr.Invalid("password must not be blank") + } + if n := utf8.RuneCountInString(plain); n < MinPasswordCodePoints || n > MaxPasswordCodePoints { + return apperr.Invalid(fmt.Sprintf("password must be %d to %d characters", MinPasswordCodePoints, MaxPasswordCodePoints)) + } + return nil +} diff --git a/internal/user/user_test.go b/internal/user/user_test.go new file mode 100644 index 00000000..ede9ec52 --- /dev/null +++ b/internal/user/user_test.go @@ -0,0 +1,78 @@ +package user_test + +import ( + "strings" + "testing" + + "messenger/internal/platform/apperr" + "messenger/internal/platform/ident" + "messenger/internal/user" +) + +func TestNewDerivesCreatedAtFromID(t *testing.T) { + id := ident.New() + u, err := user.New(id, "alice", "Алиса") + if err != nil { + t.Fatalf("New error: %v", err) + } + if u.ID != id || u.Username != "alice" || u.DisplayName != "Алиса" { + t.Fatalf("unexpected user %+v", u) + } + if !u.CreatedAt.Equal(ident.TimeOf(id)) { + t.Fatalf("CreatedAt = %v, want %v", u.CreatedAt, ident.TimeOf(id)) + } +} + +func TestNewValidatesUsername(t *testing.T) { + valid := []string{"abc", "a_1", strings.Repeat("a", 32), "user_2026"} + for _, name := range valid { + if _, err := user.New(ident.New(), name, "ok"); err != nil { + t.Errorf("username %q must be accepted: %v", name, err) + } + } + invalid := []string{"", "ab", strings.Repeat("a", 33), "Alice", "al ice", "алиса", "a-b", "a.b", " abc", "abc\n"} + for _, name := range invalid { + _, err := user.New(ident.New(), name, "ok") + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "username") { + t.Errorf("username %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} + +// The contract counts the password length in Unicode code points, not bytes. +func TestValidatePasswordCountsCodePoints(t *testing.T) { + valid := []string{ + strings.Repeat("я", 12), + strings.Repeat("👋", 128), + "pass:word:with:colons", + " spaced but long enough ", + } + for _, p := range valid { + if err := user.ValidatePassword(p); err != nil { + t.Errorf("password of %d code points must be accepted: %v", len([]rune(p)), err) + } + } + invalid := []string{"", strings.Repeat(" ", 12), strings.Repeat("\t", 20), strings.Repeat("я", 11), strings.Repeat("👋", 129)} + for _, p := range invalid { + err := user.ValidatePassword(p) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "password") { + t.Errorf("password %q must be rejected as invalid naming the field, got %v", p, err) + } + } +} + +func TestNewValidatesDisplayNameByCodePoints(t *testing.T) { + valid := []string{"A", "Алиса", strings.Repeat("я", 100), "👋 hi"} + for _, name := range valid { + if _, err := user.New(ident.New(), "alice", name); err != nil { + t.Errorf("display_name %q must be accepted: %v", name, err) + } + } + invalid := []string{"", " ", "\t\n", strings.Repeat("я", 101), "nul\x00inside"} + for _, name := range invalid { + _, err := user.New(ident.New(), "alice", name) + if apperr.KindOf(err) != apperr.KindInvalid || !strings.Contains(err.Error(), "display_name") { + t.Errorf("display_name %q must be rejected as invalid naming the field, got %v", name, err) + } + } +} diff --git a/migrations/00001_init.sql b/migrations/00001_init.sql new file mode 100644 index 00000000..400b762e --- /dev/null +++ b/migrations/00001_init.sql @@ -0,0 +1,42 @@ +-- +goose Up +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL CONSTRAINT users_username_key UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +-- Lists the chats of one user in (created_at, id) order: chat ids are UUIDv7. +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +-- The key is (chat_id, id) and serves the history query directly: +-- WHERE chat_id = $1 AND id > $2 ORDER BY id LIMIT $3. Nothing looks a +-- message up by id alone, and a separate index on id would only cost writes +-- and tempt the planner into scanning every chat's messages in id order. +CREATE TABLE messages ( + chat_id uuid NOT NULL REFERENCES chats (id), + id uuid NOT NULL, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, id) +); + +-- +goose Down +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; diff --git a/migrations/00002_message_idempotency_keys.sql b/migrations/00002_message_idempotency_keys.sql new file mode 100644 index 00000000..6fe78f6f --- /dev/null +++ b/migrations/00002_message_idempotency_keys.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method + +-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys. +CREATE TABLE message_idempotency_keys ( + user_id uuid NOT NULL, + chat_id uuid NOT NULL, + key text NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key), + FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id) +); + +CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at); + +-- +goose Down +DROP TABLE message_idempotency_keys; diff --git a/migrations/00003_attachments.sql b/migrations/00003_attachments.sql new file mode 100644 index 00000000..abb4756e --- /dev/null +++ b/migrations/00003_attachments.sql @@ -0,0 +1,35 @@ +-- +goose Up +CREATE TABLE files ( + hash bytea PRIMARY KEY CHECK (octet_length(hash) = 32), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + mime_type text NOT NULL CHECK (mime_type IN ('image/png', 'image/jpeg')), + status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + file_hash bytea NOT NULL REFERENCES files(hash), + created_at timestamptz NOT NULL +); + +-- The reconciler must distinguish a shared file from an unused file. +CREATE INDEX attachments_file_hash_idx ON attachments(file_hash); + +CREATE TABLE message_attachments ( + chat_id uuid NOT NULL, + message_id uuid NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments(id), + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY(chat_id, message_id, attachment_id), + UNIQUE(chat_id, message_id, ordinal), + FOREIGN KEY(chat_id, message_id) REFERENCES messages(chat_id, id) +); + +-- +goose Down +DROP TABLE message_attachments; +DROP TABLE attachments; +DROP TABLE files; diff --git a/migrations/00004_attachment_pipeline.sql b/migrations/00004_attachment_pipeline.sql new file mode 100644 index 00000000..eb3f0684 --- /dev/null +++ b/migrations/00004_attachment_pipeline.sql @@ -0,0 +1,79 @@ +-- +goose Up +ALTER TABLE attachments + ADD COLUMN target_version integer NOT NULL DEFAULT 1 CHECK (target_version >= 1), + ADD COLUMN published_version integer CHECK (published_version BETWEEN 1 AND target_version); + +-- A queued row with no publish timestamp is an outbox entry for this attempt. +-- attempt never decreases. The separate budget starts again only on operator replay. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')), + source_key text NOT NULL, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + budget_attempt integer NOT NULL DEFAULT 1 CHECK (budget_attempt BETWEEN 1 AND attempt), + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + lease_token uuid, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z][a-z0-9_]{0,63}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((state = 'processing') = (lease_token IS NOT NULL)), + CHECK ((state = 'processing') = (lease_until IS NOT NULL)), + CHECK (state <> 'failed' OR error_code IS NOT NULL) +); +CREATE INDEX attachment_jobs_outbox_idx ON attachment_jobs (available_at, job_id) + WHERE state='queued' AND published_at IS NULL; +CREATE INDEX attachment_jobs_recover_idx ON attachment_jobs (lease_until, job_id) WHERE state='processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at, job_id) WHERE state='failed'; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + variant text NOT NULL CHECK (variant IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL, + mime_type text NOT NULL CHECK (mime_type = CASE WHEN variant='optimized' THEN 'image/webp' ELSE 'image/jpeg' END), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + sha256 bytea NOT NULL CHECK (octet_length(sha256)=32), + PRIMARY KEY (attachment_id,pipeline_version,variant), + FOREIGN KEY (attachment_id,pipeline_version) REFERENCES attachment_jobs(attachment_id,pipeline_version) +); + +CREATE TABLE attachment_replays ( + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id), + attempt integer NOT NULL, + actor text NOT NULL CHECK (length(actor) BETWEEN 1 AND 128), + previous_error text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, attempt) +); + +-- Published Lab 4 objects are valid inputs. A recorded missing original +-- stays diagnosable even before a worker is started. +INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code) +SELECT a.id,'blobs/'||encode(f.hash,'hex'), + CASE WHEN f.status='failed' THEN 'failed' ELSE 'queued' END, + CASE WHEN f.status='failed' THEN 'source_missing' END +FROM attachments a JOIN files f ON f.hash=a.file_hash; + +CREATE VIEW attachment_states AS +SELECT a.id, + CASE WHEN a.published_version IS NULL THEN j.state + WHEN f.status='failed' THEN 'failed' ELSE 'ready' END AS status, + CASE WHEN a.published_version IS NOT NULL AND f.status='failed' THEN 'source_missing' + WHEN a.published_version IS NULL AND j.state='failed' THEN j.error_code END AS error_code, + j.updated_at +FROM attachments a JOIN files f ON f.hash=a.file_hash +JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.target_version; + +-- +goose Down +DROP VIEW attachment_states; +DROP TABLE attachment_replays; +DROP TABLE attachment_variants; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN published_version, DROP COLUMN target_version; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 00000000..29af9ab0 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +// Package migrations embeds the versioned SQL schema, so the server binary +// always carries the schema it was built against. +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS diff --git a/requirements-bench.txt b/requirements-bench.txt new file mode 100644 index 00000000..b1ec8530 --- /dev/null +++ b/requirements-bench.txt @@ -0,0 +1,2 @@ +# Plotting of the track 1 results (scripts/bench/plot.py); install into a venv. +matplotlib==3.10.3 diff --git a/scripts/api-instances.sh b/scripts/api-instances.sh new file mode 100755 index 00000000..be6ce307 --- /dev/null +++ b/scripts/api-instances.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Lists the API containers: . The +# alias is random per process, so each running replica is asked over +# /health/live, in parallel: the failover hook must return within 10 seconds. +set -eu +cd "$(dirname "$0")/.." + +ids=$(docker compose ps -aq api) +[ -n "$ids" ] || exit 0 +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +docker inspect --format '{{.Id}} {{.Name}} {{.State.Status}} {{.Image}}' $ids >"$tmp/containers" +while read -r id name state image; do + if [ "$state" = running ]; then + # wget prints the response headers on stderr; the body is dropped. + (docker exec "$id" wget -S -q -O - http://127.0.0.1:8080/health/live 2>&1 >/dev/null | + sed -n 's/^ *[Xx]-[Ii]nstance-[Ii]d: *//p' | tr -d '\r' | head -n 1 >"$tmp/$id") & + fi +done <"$tmp/containers" +wait + +while read -r id name state image; do + alias=- + if [ -s "$tmp/$id" ]; then + alias=$(cat "$tmp/$id") + fi + image=${image#sha256:} + printf '%s %s %s %s %s\n' "$(printf '%s' "$id" | cut -c1-12)" "${name#/}" "$state" "$alias" "$(printf '%s' "$image" | cut -c1-12)" +done <"$tmp/containers" diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh new file mode 100755 index 00000000..afff54d9 --- /dev/null +++ b/scripts/bench/failover.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and +# started again at RESTORE_AT, under a constant write load that retries +# uncertain sends with the same key; then every confirmed message must be in +# the history once. +# +# sh scripts/bench/failover.sh stop|kill +# +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT +# (evidence/failover). Times come from the Docker VM clock; the Windows clock +# differs by a fraction of a second and is kept for reference only. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +kind=${1:?usage: failover.sh stop|kill } +n=${2:?usage: failover.sh stop|kill } +case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac +rate=${RATE:-50} +duration=${DURATION:-60s} +fault_at=${FAULT_AT:-15} +restore_at=${RESTORE_AT:-35} +dir=${OUT:-evidence/failover}/$kind-$n +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +target=$(docker compose ps -q api | sed -n "${n}p") +[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; } +alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }') +meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ + "fault_at_s=$fault_at" "restore_at_s=$restore_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$fault_at" +fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker "$kind" "$target" >/dev/null +exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target") +sleep $((restore_at - fault_at)) +restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker compose start api >/dev/null 2>&1 +started=$(docker inspect --format '{{.State.StartedAt}}' "$target") +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json" +docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh new file mode 100644 index 00000000..8f15fd23 --- /dev/null +++ b/scripts/bench/lib.sh @@ -0,0 +1,77 @@ +# Shared helpers of the experiment scripts; sourced, not run. + +# The generator and the knobs the experiments vary live in compose.bench.yaml. +export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml +# The balancer logs errors and state changes only, not every request. +export LB_QUIET=1 + +# abspath prints a directory in the form the Docker CLI mounts (a Windows path +# under Git Bash). +abspath() { + (cd "$1" && (pwd -W 2>/dev/null || pwd)) +} + +# meta writes what ran, when, on which commit and host: file, then key=value. +meta() { + file=$1 + shift + { + printf '{\n' + printf ' "date_utc": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf ' "commit": "%s",\n' "$(git rev-parse --short HEAD)" + printf ' "branch": "%s",\n' "$(git branch --show-current)" + printf ' "docker": "%s",\n' "$(docker version --format '{{.Server.Version}}')" + printf ' "host_cpus": "%s",\n' "$(docker info --format '{{.NCPU}}')" + printf ' "host_memory_bytes": "%s",\n' "$(docker info --format '{{.MemTotal}}')" + for pair in "$@"; do + printf ' "%s": "%s",\n' "${pair%%=*}" "${pair#*=}" + done + printf ' "go": "%s"\n' "$(go env GOVERSION 2>/dev/null || echo unknown)" + printf '}\n' + } >"$file" +} + +# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: +# the same work every time, so a slow spell of the host shows in the results. +host_probe() { + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab5 \ + -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' +} + +# state_log keeps from `compose logs` (stdin) what explains a run: the +# balancer's server state changes and the replicas' start and stop. An +# overloaded run logs every failed request, gigabytes over a series; those +# lines are only counted. +state_log() { + awk '/is DOWN|is UP|UP\/READY|maintenance|"msg":"(starting|shutdown started|shutdown completed)"/ { print; next } + { dropped++ } END { printf "# %d other lines dropped\n", dropped }' +} + +# sample_docker_stats appends the CPU and memory of every container until killed. +sample_docker_stats() { + while :; do + docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ + $(docker ps -q --filter label=com.docker.compose.project=lab5) 2>/dev/null || true + done >>"$1" +} + +# sample_postgres appends, every second, the database counters and what the +# active sessions wait for ("cpu": nothing). +sample_postgres() { + while :; do + docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " + SELECT json_build_object( + 't', extract(epoch FROM now()), + 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, + 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'sessions', sessions, 'sessions_abandoned', sessions_abandoned, + 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), + 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( + SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n + FROM pg_stat_activity + WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid() + GROUP BY 1) AS x)) + FROM pg_stat_database WHERE datname = 'messenger'" 2>/dev/null || true + sleep 1 + done >>"$1" +} diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py new file mode 100755 index 00000000..5258974c --- /dev/null +++ b/scripts/bench/plot.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Tables and charts of track 1 from the raw results of run-series.sh. + + python scripts/bench/plot.py + +The raw root holds //rep/ directories. Written to the +output directory: + summary.csv one row per step of every run; + capacity.csv the capacity of every run and the median per configuration; + control.csv the noise checks of every run, the host speed probe and the + loadgen CPU per request at the first step, each flagged when + more than 25 % off its median; + cost.csv CPU milliseconds per request of each component at the + COST_RATES steps (valid, error-free, unflagged runs only), and + the rate one database CPU would serve at that cost; + -throughput.png, -p99.png, capacity.png, + cpu-at-capacity.png, pool-waits.png, cost-per-request.png. +Capacity is the highest step that passed the criterion loadgen applies (valid, +under 1 % errors, p99 of successful requests within 500 ms). Charts show the +median of the repeats; a hollow marker means the generator fell behind. +""" +import csv +import gzip +import json +import pathlib +import statistics +import sys +from datetime import datetime + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +SLO_P99_MS = 500.0 +CONTROL_TOLERANCE = 0.25 +# Below saturation in every configuration, so the cost is not inflated by +# queueing. +COST_RATES = (800, 1000) +COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") + + +def read_jsonl(path): + packed = path.with_name(path.name + ".gz") + if packed.exists(): + text = gzip.decompress(packed.read_bytes()).decode("utf-8") + elif path.exists(): + text = path.read_text(encoding="utf-8") + else: + return [] + rows = [] + for line in text.splitlines(): + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass + return rows + + +def component(name): + for c in COMPONENTS: + if f"-{c}-" in name: + return c + return None + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def load_runs(root): + runs = [] + for steps_path in sorted(root.glob("*/*/rep*/steps.json")): + d = steps_path.parent + report = json.loads(steps_path.read_text(encoding="utf-8")) + runs.append({ + "series": d.parent.parent.name, + "config": d.parent.name, + "rep": d.name, + "start": parse_time(report["started_at"]), + "steps": report["steps"], + "docker": read_jsonl(d / "docker-stats.jsonl"), + "metrics": read_jsonl(d / "metrics.jsonl"), + "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, + }) + return runs + + +def mean_cpu(run, t0, t1): + """Mean CPU per component in [t0, t1); replicas of the API are summed.""" + per_time = {} + for row in run["docker"]: + c = component(row.get("name", "")) + if c is None or not t0 <= row["t"] < t1: + continue + try: + cpu = float(row["cpu"].rstrip("%")) + except (ValueError, KeyError): + continue + per_time.setdefault(row["t"], {}).setdefault(c, 0.0) + per_time[row["t"]][c] += cpu + out = {} + for c in COMPONENTS: + values = [v[c] for v in per_time.values() if c in v] + out[c] = round(statistics.mean(values), 1) if values else "" + return out + + +def pool_rate(run, t0, t1, counter): + """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" + first, last = {}, {} + for row in run["metrics"]: + t = run["start"] + row["t"] + if not t0 <= t < t1: + continue + for replica in row.get("replicas") or []: + pool = replica.get("pgxpool") or {} + key = replica.get("instance") + if key and counter in pool: + first.setdefault(key, pool[counter]) + last[key] = pool[counter] + if not last: + return "" + return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) + + +def main(root, out): + runs = load_runs(root) + if not runs: + sys.exit(f"no runs under {root}") + out.mkdir(parents=True, exist_ok=True) + + rows, capacities = [], [] + for run in runs: + best = None + for step in run["steps"]: + t0 = run["start"] + step["started_at_s"] + t1 = t0 + step["seconds"] + cpu = mean_cpu(run, t0, t1) + classes = step["classes"] + row = { + "series": run["series"], "config": run["config"], "rep": run["rep"], + "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), + "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], + "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], + "client_timeouts": classes.get("client_timeout", 0), + "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), + "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), + **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, + "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), + "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), + } + rows.append(row) + if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): + best = row + capacities.append({ + "series": run["series"], "config": run["config"], "rep": run["rep"], + "capacity_rps": best["offered_rps"] if best else 0, "at": best, + "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), + }) + + with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + configs = [] + for c in capacities: + if (c["series"], c["config"]) not in configs: + configs.append((c["series"], c["config"])) + with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) + medians = {} + for key in configs: + mine = [c for c in capacities if (c["series"], c["config"]) == key] + values = [c["capacity_rps"] for c in mine] + medians[key] = statistics.median(values) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", + sum(c["invalid_below"] for c in mine)]) + + control = [] + for run in runs: + first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] + if first and first[0]["cpu_loadgen_pct"] != "": + probe = run["meta"].get("host_probe_s", "") + control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, + round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) + if control: + probes = [c[3] for c in control if c[3] is not None] + probe_median = statistics.median(probes) if probes else None + cpu_median = statistics.median(c[4] for c in control) + with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", + "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) + flagged = set() + for series, config, rep, probe, cpu in control: + probe_dev = probe / probe_median - 1 if probe is not None else 0.0 + cpu_dev = cpu / cpu_median - 1 + noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE + if noisy: + flagged.add((series, config, rep)) + w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, + f"{cpu_dev:+.2f}", noisy]) + + cost = {} + for r in rows: + if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES: + continue + if not r["valid"] or r["error_share"] > 0: + continue + per = cost.setdefault((r["series"], r["config"]), {"runs": set()}) + per["runs"].add(r["rep"]) + for c in ("api", "postgres", "lb"): + if r[f"cpu_{c}_pct"] != "": + per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"]) + with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request", + "lb_cpu_ms_per_request", "rps_per_database_cpu"]) + for (series, config), per in cost.items(): + med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)} + w.writerow([series, config, " ".join(sorted(per["runs"])), + *(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")), + round(1000 / med["postgres"]) if med.get("postgres") else ""]) + if cost: + keys = list(cost) + fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5)) + width = 0.27 + for i, c in enumerate(("api", "postgres", "lb")): + ax.bar([j + i * width for j in range(len(keys))], + [statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c) + ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30) + ax.set_ylabel("CPU ms per request") + ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cost-per-request.png", dpi=120) + plt.close(fig) + + def median_by_offered(series, config, field): + points, valid = {}, {} + for r in rows: + if r["series"] == series and r["config"] == config and r[field] != "": + points.setdefault(r["offered_rps"], []).append(float(r[field])) + valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] + xs = sorted(points) + return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] + + for series in sorted({s for s, _ in configs}): + names = [c for s, c in configs if s == series] + for field, title, filename, log in ( + ("ok_rps", "Successful requests per second", "throughput", False), + ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), + ): + fig, ax = plt.subplots(figsize=(7, 4.5)) + top = 0 + for name in names: + xs, ys, valid = median_by_offered(series, name, field) + line, = ax.plot(xs, ys, label=name) + ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) + ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], + facecolors="none", edgecolors=line.get_color()) + top = max([top, *xs]) + if field == "ok_rps" and top: + ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") + if field == "ok_p99_ms": + ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") + if log: + ax.set_yscale("log") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel(title) + ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / f"{series}-{filename}.png", dpi=120) + plt.close(fig) + + labels = [f"{s}/{c}" for s, c in configs] + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) + meds = [medians[k] for k in configs] + lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] + highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] + ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") + ax.set_ylabel("Capacity, requests per second") + ax.set_title("Capacity per configuration (median, range of repeats)") + ax.tick_params(axis="x", rotation=30) + ax.grid(axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(out / "capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) + width = 0.8 / len(COMPONENTS) + for i, comp in enumerate(COMPONENTS): + values = [] + for key in configs: + at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] + cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] + values.append(statistics.median(cpu) if cpu else 0) + ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) + ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) + ax.set_ylabel("Mean CPU at capacity, % of one core") + ax.set_title("Which component is busy at capacity") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cpu-at-capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(7, 4.5)) + for series, name in configs: + xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") + if xs: + ax.plot(xs, ys, marker="o", label=f"{series}/{name}") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel("Waits for a pooled connection per second") + ax.set_title("Connection pool waits, all replicas") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "pool-waits.png", dpi=120) + plt.close(fig) + print(f"{len(runs)} runs, {len(rows)} steps -> {out}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])) diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh new file mode 100755 index 00000000..44cfb40b --- /dev/null +++ b/scripts/bench/rollout-under-load.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a +# constant write load that retries uncertain sends with the same key; then +# every confirmed message must be in the history once. +# +# sh scripts/bench/rollout-under-load.sh +# +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout). +# Times come from the Docker VM clock, as in failover.sh. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +rate=${RATE:-50} +duration=${DURATION:-60s} +update_at=${UPDATE_AT:-15} +dir=${OUT:-evidence/rollout} +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +# Built beforehand, as by a delivery pipeline: the update replaces replicas. +docker compose build api >/dev/null 2>&1 +meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) +old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ') + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$update_at" +started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +status=0 +sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? +finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +new="" +for id in $(docker compose ps -q api); do + new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, " +done +printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \ + "$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json" +docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh new file mode 100755 index 00000000..f291fa77 --- /dev/null +++ b/scripts/bench/run-series.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# Track 1: every configuration of the given series files, each on a fresh +# stand, under one grid of offered rates. Repeats are interleaved (repeat 1 +# of every configuration, then repeat 2), so a slow spell of the host does not +# fall on one configuration. +# +# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...] +# +# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s), +# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT +# (evidence/track1/raw). Results: ///rep/. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh ..." >&2; exit 2; } +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}" +repeats=${REPEATS:-3} +step=${STEP:-30s} +warmup=${WARMUP:-30s} +out=${OUT:-evidence/track1/raw} +stop= +[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail + +# Fixed for every configuration; the API, its pool, maxconn and the database +# CPU come from the series line. +export LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 +export MSYS_NO_PATHCONV=1 + +# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus. +run_config() { + series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10} + dir=$out/$series/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \ + DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus + + docker compose down -v --remove-orphans >/dev/null 2>&1 + if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then + echo " the stand did not start, see $dir/up.log" >&2 + exit 1 + fi + meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \ + "db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \ + "lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \ + "stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)" + + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + # stdin closed: `compose run` would otherwise read it. + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 /dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + docker compose logs --no-color --timestamps lb api 2>&1 | state_log >"$dir/compose.log" + gzip -9 "$dir"/*.jsonl + grep '^step' "$dir/loadgen.log" || true +} + +for conf in "$@"; do + [ -r "$conf" ] || { echo "cannot read $conf" >&2; exit 2; } + bad=$(grep -v '^#' "$conf" | awk 'NF && NF != 8') + [ -z "$bad" ] || { echo "$conf: a line needs 8 fields: $bad" >&2; exit 2; } +done +docker compose --profile bench build loadgen >/dev/null +default_ifs=$IFS +newline=' +' +for rep in $(seq 1 "$repeats"); do + for conf in "$@"; do + lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d') + IFS=$newline + for line in $lines; do + IFS=$default_ifs + # shellcheck disable=SC2086 # the line splits into the fields + run_config "$(basename "$conf" .conf)" "$rep" $line + done + IFS=$default_ifs + done +done +docker compose down -v --remove-orphans >/dev/null 2>&1 +echo "results in $out" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf new file mode 100644 index 00000000..a461f2b3 --- /dev/null +++ b/scripts/bench/series/A.conf @@ -0,0 +1,6 @@ +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas; +# the pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A1 1 2.0 1024M 920MiB 10 0 1.0 +A2 2 1.0 512M 460MiB 10 0 1.0 +A4 4 0.5 256M 230MiB 10 0 1.0 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf new file mode 100644 index 00000000..6ebe2902 --- /dev/null +++ b/scripts/bench/series/Aprime.conf @@ -0,0 +1,5 @@ +# Series A': the budget of series A with the total pool fixed at 20 connections; +# A'2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +Ap1 1 2.0 1024M 920MiB 20 0 1.0 +Ap4 4 0.5 256M 230MiB 5 0 1.0 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf new file mode 100644 index 00000000..9f4fc11c --- /dev/null +++ b/scripts/bench/series/B.conf @@ -0,0 +1,5 @@ +# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +B1 1 1.0 512M 460MiB 10 0 1.0 +B3 3 1.0 512M 460MiB 10 0 1.0 +B4 4 1.0 512M 460MiB 10 0 1.0 diff --git a/scripts/bench/series/C.conf b/scripts/bench/series/C.conf new file mode 100644 index 00000000..20d012f3 --- /dev/null +++ b/scripts/bench/series/C.conf @@ -0,0 +1,10 @@ +# Improvement C: maxconn per replica on the balancer, before (0, no limit) and +# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0); +# 10 and 40 show how much the value matters. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +R1m0 1 1.0 512M 460MiB 10 0 1.0 +R1m20 1 1.0 512M 460MiB 10 20 1.0 +R4m0 4 1.0 512M 460MiB 10 0 1.0 +R4m20 4 1.0 512M 460MiB 10 20 1.0 +R1m10 1 1.0 512M 460MiB 10 10 1.0 +R1m40 1 1.0 512M 460MiB 10 40 1.0 diff --git a/scripts/bench/series/control.conf b/scripts/bench/series/control.conf new file mode 100644 index 00000000..76a08eec --- /dev/null +++ b/scripts/bench/series/control.conf @@ -0,0 +1,6 @@ +# Control: A2 next to A2 with twice the database CPU, interleaved, run with +# STOP_ON_FAIL=0 so that one noisy step does not end a ladder. If the database +# is the bottleneck, the capacity rises. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A2 2 1.0 512M 460MiB 10 0 1.0 +A2pg2 2 1.0 512M 460MiB 10 0 2.0 diff --git a/scripts/explain-history.sql b/scripts/explain-history.sql new file mode 100644 index 00000000..f455eedd --- /dev/null +++ b/scripts/explain-history.sql @@ -0,0 +1,75 @@ +-- EXPLAIN evidence for the message history query (make explain). +-- +-- Everything runs in one transaction that is rolled back: it seeds 12 000 +-- messages in one chat and 90 000 in 300 other chats, so the plan is measured +-- on a realistic volume without leaving data on the stand. The PREPARE text +-- is the statement of Messages.ListByChat in internal/chat/store/postgres.go; +-- keep them identical. +\set ON_ERROR_STOP on +\pset pager off +BEGIN; + +SELECT uuidv7() AS author \gset +SELECT uuidv7() AS big_chat \gset + +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES (:'author', 'explain_' || substr(md5(random()::text), 1, 12), 'EXPLAIN', 'not-a-password-hash', + uuid_extract_timestamp(:'author')); + +CREATE TEMP TABLE noise_chats ON COMMIT DROP AS +SELECT uuidv7() AS id FROM generate_series(1, 300); + +INSERT INTO chats (id, title, created_at) +SELECT :'big_chat'::uuid, 'explain: big chat', uuid_extract_timestamp(:'big_chat') +UNION ALL +SELECT id, 'explain: other chat', uuid_extract_timestamp(id) FROM noise_chats; + +INSERT INTO chat_members (chat_id, user_id) +SELECT id, :'author' FROM chats WHERE title LIKE 'explain:%'; + +-- Rows are inserted ordered by n, so the messages of all chats interleave in +-- the heap as they would under real traffic; ids stay increasing per chat. +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT id, chat_id, :'author', 'message ' || n, uuid_extract_timestamp(id) +FROM ( + SELECT uuidv7() AS id, chat_id, n + FROM ( + SELECT :'big_chat'::uuid AS chat_id, n FROM generate_series(1, 12000) AS n + UNION ALL + SELECT c.id, n FROM noise_chats AS c, generate_series(1, 300) AS n + ) AS planned + ORDER BY n +) AS generated; + +ANALYZE users, chats, chat_members, messages; + +SELECT (SELECT count(*) FROM messages WHERE chat_id = :'big_chat') AS big_chat_messages, + (SELECT count(*) FROM messages) AS all_messages; + +PREPARE history(uuid, uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 + AND id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY id +LIMIT $3; + +\echo +\echo == First page: no cursor, limit 50 (+1 row that tells whether a next page exists) +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', NULL, 51); + +SELECT id AS deep_cursor FROM messages WHERE chat_id = :'big_chat' ORDER BY id OFFSET 10000 LIMIT 1 \gset +\echo +\echo == Deep page: cursor after the 10 001st message of the chat +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history(:'big_chat', :'deep_cursor', 51); + +\echo +\echo == For contrast, OFFSET pagination of the same page reads every skipped row +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = :'big_chat' +ORDER BY id +OFFSET 10001 LIMIT 51; + +ROLLBACK; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..7bc9f22b --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,105 @@ +#!/bin/sh +# Creates the local credentials of the compose stand in ./secrets, which git +# ignores. Existing secrets are kept: PostgreSQL applies its password only +# when the volume is first initialised, so a new password would lock the API +# out of an existing database. To start over: make destroy, then delete +# ./secrets and run this again. +set -eu +cd "$(dirname "$0")/.." + +files="postgres_password database_url valkey_auth.conf session_store_url" +present=0 +for f in $files; do + if [ -e "secrets/$f" ]; then + present=$((present + 1)) + fi +done +if [ "$present" -ne 0 ] && [ "$present" -ne 4 ]; then + echo "./secrets is incomplete: delete it (and run make destroy if the stand holds data), then run again" >&2 + exit 1 +fi + +# 24 random bytes as hex: URL-safe without escaping. +random() { + od -An -tx1 -N24 /dev/urandom | tr -d ' \n' +} +# The directory is private; the files stay readable for the unprivileged +# users inside the containers, which see them through bind mounts. +umask 022 +mkdir -p secrets +chmod 700 secrets +if [ "$present" -eq 0 ]; then + pg=$(random) + kv=$(random) + printf '%s\n' "$pg" > secrets/postgres_password + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$pg" > secrets/database_url + printf 'requirepass %s\n' "$kv" > secrets/valkey_auth.conf + printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url +fi + +if [ ! -e secrets/rabbitmq.conf ] && [ ! -e secrets/broker_url ]; then + rabbit_password=$(random) + printf 'default_user = messenger\ndefault_pass = %s\n' "$rabbit_password" > secrets/rabbitmq.conf + printf 'amqp://messenger:%s@broker:5672/\n' "$rabbit_password" > secrets/broker_url +elif [ ! -s secrets/rabbitmq.conf ] || [ ! -s secrets/broker_url ]; then + echo "restore rabbitmq.conf and broker_url together" >&2 + exit 1 +fi + +# Read just the bucket setting, never execute the local .env file as a script. +bucket=${S3_BUCKET:-} +if [ -z "$bucket" ] && [ -f .env ]; then + bucket=$(sed -n 's/^S3_BUCKET=//p' .env | tail -n 1 | tr -d '\r') +fi +bucket=${bucket:-messenger-attachments} +case "$bucket" in + *[!a-z0-9.-]*|.*|*.|-*|*-|*..*|*.-*|*-.*) + echo "S3_BUCKET must be a lowercase S3 bucket name" >&2; exit 1 ;; +esac +if [ "${#bucket}" -lt 3 ] || [ "${#bucket}" -gt 63 ]; then + echo "S3_BUCKET must contain 3 to 63 characters" >&2; exit 1 +fi + +s3files="s3_admin_access_key_id s3_admin_secret_access_key s3_access_key_id s3_secret_access_key s3_reconcile_access_key_id s3_reconcile_secret_access_key s3_identities.json" +s3present=0 +for f in $s3files; do + if [ -s "secrets/$f" ]; then s3present=$((s3present + 1)); fi +done +if [ "$s3present" -eq 7 ]; then + if ! grep -Fq "\"Read:$bucket\"" secrets/s3_identities.json; then + echo "existing S3 identities use another bucket; keep S3_BUCKET consistent with the retained volume" >&2 + exit 1 + fi + echo "existing PostgreSQL, Valkey and S3 secrets kept" + exit 0 +fi +if [ "$s3present" -ne 0 ]; then + echo "S3 secrets are incomplete; restore the s3_* secret files together; PostgreSQL and Valkey secrets were kept" >&2 + exit 1 +fi + +admin_key=$(random) +admin_secret=$(random) +api_key=$(random) +api_secret=$(random) +recon_key=$(random) +recon_secret=$(random) +printf '%s\n' "$admin_key" > secrets/s3_admin_access_key_id +printf '%s\n' "$admin_secret" > secrets/s3_admin_secret_access_key +printf '%s\n' "$api_key" > secrets/s3_access_key_id +printf '%s\n' "$api_secret" > secrets/s3_secret_access_key +printf '%s\n' "$recon_key" > secrets/s3_reconcile_access_key_id +printf '%s\n' "$recon_secret" > secrets/s3_reconcile_secret_access_key + +# The server loads all identities before listening. There is no anonymous +# identity, and no post-start allow-all provisioning phase. +cat > secrets/s3_identities.json <=2 GiB limits. + +Run with python -X utf8 -B. Default is one upload; no container lifecycle actions. +""" +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import struct +import subprocess +import sys +import threading +import time +from urllib.parse import urlsplit +import zlib + +sys.dont_write_bytecode = True +_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) +memory = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(memory) + + +def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536): + """Stream zero scanlines, including filter bytes; never allocate a full row.""" + if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000: + raise ValueError('invalid stress dimensions or chunk size') + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + encoder = zlib.compressobj(1) + remaining = height * (width * 8 + 1) + zeros = bytes(min(chunk_bytes, remaining)) + with path.open('wb') as out: + out.write(b'\x89PNG\r\n\x1a\n') + out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0))) + while remaining: + count = min(len(zeros), remaining) + compressed = encoder.compress(zeros[:count]) + if compressed: + out.write(memory.png_chunk(b'IDAT', compressed)) + remaining -= count + out.write(memory.png_chunk(b'IDAT', encoder.flush())) + out.write(memory.png_chunk(b'IEND', b'')) + + +def fixture_info(content, mime): + if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33: + width, height, depth, color = struct.unpack_from('>IIBB', content, 16) + return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color} + if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'): + pos = 2 + while pos + 4 <= len(content): + if content[pos] != 255: + break + while pos < len(content) and content[pos] == 255: + pos += 1 + marker = content[pos] + pos += 1 + length = struct.unpack_from('>H', content, pos)[0] + frame = content[pos + 2:pos + length] + pos += length + if marker in (0xc0, 0xc1, 0xc2): + height, width = struct.unpack_from('>HH', frame, 1) + count = frame[5] + return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count, + 'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]} + if length < 2 or marker == 0xda: + break + raise ValueError('fixture MIME/header mismatch or missing image header') + + +def read_fixture(path, mime): + if not 0 < path.stat().st_size <= 10 * 1024 * 1024: + raise ValueError('fixture must be between 1 byte and 10 MiB') + content = path.read_bytes() + info = fixture_info(content, mime) + if not 0 < info['width'] * info['height'] <= 40_000_000: + raise ValueError('fixture exceeds 40M pixels') + return content, info + + +def api_limits(ids): + template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}' + result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())} + + +def experiment(args, content, report): + sys.path.insert(0, str(memory.ROOT / 'tests')) + import client as module + ids = memory.api_containers(args) + report['api_limits'] = api_limits(ids) + if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids): + raise RuntimeError('stress uploads require every API container limit to be >=2 GiB') + report['state_before'] = memory.container_states(ids) + bootstrap = module.Client(args.base_url, timeout=args.timeout) + actor, reader, _, chat = bootstrap.bootstrap() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + with memory.DockerStats(ids) as stats: + stats.mark(args.label, 'baseline') + stats.wait_for_baseline(args.label) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'], + barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)] + stats.mark(args.label, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + report['duration_s'] = time.monotonic() - started + time.sleep(args.tail_seconds) + stats.mark(args.label, 'idle') + report['memory_samples'] = stats.snapshot(args.label) + report['sampling_error'] = stats.error + report['state_after'] = memory.container_states(ids) + stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed'] + and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids) + report['restart_or_oom_detected'] = not stable + try: + report['memory'] = memory.summarize_memory(report['memory_samples'], ids) + except RuntimeError as exc: + report['memory_error'] = str(exc) + report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results]) + accepted = [r for r in results if r['ok']] + report['accepted'] = len(accepted) + report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)) + report['latency'] = memory.latency_summary([r['latency_ms'] for r in results]) + report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + report['results'] = results + report['download'] = {'ok': False} + if accepted: + try: + report['download'] = memory.verify_download(bootstrap, reader, accepted[0]['attachment_id'], content, args.mime) + except Exception as exc: + report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__ + report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable + and not report.get('memory_error') and not report['sampling_error'] + and report['observed_max_inflight'] == args.concurrency) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--file', type=Path) + parser.add_argument('--mime', choices=('image/png', 'image/jpeg')) + parser.add_argument('--label', default='decoder') + parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml') + parser.add_argument('--project-name') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--output', type=Path) + parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker') + parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input') + args = parser.parse_args() + if args.generate_wide: + write_rgba64_png(args.generate_wide) + content, info = read_fixture(args.generate_wide, 'image/png') + print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})) + return 0 + if not args.file or not args.mime: + parser.error('--file and --mime are required') + if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s') + if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label): + parser.error('label must be 1-40 safe ASCII characters') + url = urlsplit(args.base_url) + if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment: + parser.error('base URL must be HTTP(S) without credentials/query/fragment') + content, info = read_fixture(args.file, args.mime) + report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info, + 'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(), + 'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'measurement': {'source': 'docker stats JSON stream, formatted working-set values', + 'limitation': 'sampled/rounded peaks can miss instantaneous maxima', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}} + if args.check_file: + print(json.dumps(report, indent=2)) + return 0 + if not args.output: + parser.error('--output is required for a live run') + try: + experiment(args, content, report) + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure': + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/lab4/failover.py b/scripts/lab4/failover.py new file mode 100644 index 00000000..a16b81f2 --- /dev/null +++ b/scripts/lab4/failover.py @@ -0,0 +1,95 @@ +"""Stop each actual uploader replica and verify its bytes through the survivor.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def instances(): + text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT, + encoding='utf-8') + result = [] + for line in text.splitlines(): + container, replica, state, alias, image = line.split() + result.append({'container': container, 'replica': replica, + 'state': state, 'alias': alias, 'image': image}) + return result + + +def restore(c): + subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT, + check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id'] + for _ in range(6)} + if len(observed) >= 2: + return + time.sleep(0.3) + raise AssertionError('two replicas did not return after restore') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json') + args = p.parse_args() + c = Client(args.base_url, timeout=10) + actor, reader, _, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + wanted = instances() + check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted), + 'this experiment requires two running API replicas') + report = {'scenario': 'stop actual upload-serving replica; download via survivor', + 'cases': [], 'ok': False} + try: + for target in wanted: + upload = None + for _ in range(10): + attempt = c.call('POST', '/api/v1/chats/' + chat['id'] + + '/attachments?filename=sample.png', 201, + actor=actor, raw=data, + headers={'Content-Type': 'image/png'}, schema='Attachment') + if attempt['headers']['X-Instance-Id'] == target['alias']: + upload = attempt + break + check(upload is not None, 'target replica did not serve an upload') + before = instances() + env = dict(os.environ, TARGET_INSTANCE_ID=target['alias']) + started = time.monotonic() + subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env, + check=True, stdout=subprocess.DEVNULL, timeout=15) + stop_seconds = time.monotonic() - started + downloaded = c.call('GET', '/api/v1/attachments/' + + upload['json']['id'] + '/content', actor=reader) + survivor = downloaded['headers']['X-Instance-Id'] + check(survivor != target['alias'], 'stopped uploader still served download') + check(downloaded['bytes'] == data, 'bytes changed after uploader stop') + after = instances() + check(any(x['container'] == target['container'] and x['state'] != 'running' + for x in after), 'uploader was not left stopped') + report['cases'].append({'uploader': target, 'download_instance': survivor, + 'stop_seconds': round(stop_seconds, 3), + 'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(), + 'same_bytes': True, 'before': before, 'after': after}) + restore(c) + report['ok'] = True + finally: + # This script only restores the API containers that it deliberately stopped. + restore(c) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']), + 'output': str(args.output)})) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/generate-decoder-fixtures.py b/scripts/lab4/generate-decoder-fixtures.py new file mode 100644 index 00000000..dfc752ba --- /dev/null +++ b/scripts/lab4/generate-decoder-fixtures.py @@ -0,0 +1,45 @@ +"""Generate optional 40 MP regression images; requires Pillow==11.1.0.""" +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import sys + +sys.dont_write_bytecode = True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output-dir', type=Path, default=Path('bin/decoder-fixtures')) + args = parser.parse_args() + try: + from PIL import Image + except ImportError: + parser.error('install optional fixture dependency: pip install Pillow==11.1.0') + spec = importlib.util.spec_from_file_location('decoder_memory', + Path(__file__).with_name('decoder-memory.py')) + decoder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(decoder) + args.output_dir.mkdir(parents=True, exist_ok=True) + manifest = {'pillow_version': Image.__version__, 'fixtures': []} + for mode, color in [('RGB', (25, 90, 160)), ('CMYK', (20, 40, 60, 10))]: + path = args.output_dir / ('progressive-' + mode.lower() + '-444-40mp.jpg') + with Image.new(mode, (8000, 5000), color) as image: + image.save(path, format='JPEG', progressive=True, subsampling=0, quality=85) + content, info = decoder.read_fixture(path, 'image/jpeg') + if not info['progressive'] or any(value != 0x11 for value in info['sampling']): + raise ValueError('encoder did not produce progressive 4:4:4') + manifest['fixtures'].append({'name': path.name, 'mime': 'image/jpeg', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + wide = args.output_dir / 'wide-rgba64-40mp.png' + decoder.write_rgba64_png(wide) + content, info = decoder.read_fixture(wide, 'image/png') + manifest['fixtures'].append({'name': wide.name, 'mime': 'image/png', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + (args.output_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8') + print(json.dumps(manifest, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/test_decoder_memory.py b/scripts/lab4/test_decoder_memory.py new file mode 100644 index 00000000..45e198ef --- /dev/null +++ b/scripts/lab4/test_decoder_memory.py @@ -0,0 +1,55 @@ +"""Offline checks for bounded stress fixtures and format identification.""" +import importlib.util +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +class DecoderMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = Path(__file__).with_name('decoder-memory.py') + cls.module = None + if path.exists(): + spec = importlib.util.spec_from_file_location('decoder_memory', path) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def setUp(self): + self.assertIsNotNone(self.module, 'decoder-memory.py is not implemented') + + def test_streamed_rgba64_png_has_valid_crc_and_zero_pixels(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'wide.png' + self.module.write_rgba64_png(path, width=19, height=3, chunk_bytes=7) + payload = path.read_bytes() + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, body = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + body) & 0xffffffff) + if kind == b'IDAT': + compressed.extend(body) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(3 * (19 * 8 + 1))) + self.assertEqual(self.module.fixture_info(payload, 'image/png'), + {'width': 19, 'height': 3, 'bit_depth': 16, 'color_type': 6}) + + def test_progressive_jpeg_444_components_are_identified(self): + sof = b'\x08' + struct.pack('>HHB', 5000, 8000, 3) + bytes([1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]) + payload = b'\xff\xd8\xff\xc2' + struct.pack('>H', len(sof) + 2) + sof + info = self.module.fixture_info(payload, 'image/jpeg') + self.assertEqual((info['width'], info['height'], info['components']), (8000, 5000, 3)) + self.assertTrue(info['progressive']) + self.assertEqual(info['sampling'], [0x11, 0x11, 0x11]) + + def test_fixture_byte_cap_is_checked_before_reading(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'oversized.png' + with path.open('wb') as out: + out.truncate(10 * 1024 * 1024 + 1) + with self.assertRaises(ValueError): + self.module.read_fixture(path, 'image/png') diff --git a/scripts/lab4/test_upload_memory.py b/scripts/lab4/test_upload_memory.py new file mode 100644 index 00000000..4c36d4f9 --- /dev/null +++ b/scripts/lab4/test_upload_memory.py @@ -0,0 +1,164 @@ +"""Offline checks for reproducible PNGs and Docker memory evidence.""" +import importlib.util +import json +from pathlib import Path +import struct +import unittest +from types import SimpleNamespace +import io +from email.message import Message +from unittest.mock import Mock +import threading +import zlib + + +class UploadMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def inspect_png(self, payload, width, height): + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, chunks, compressed = 8, [], bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind = payload[pos + 4:pos + 8] + data = payload[pos + 8:pos + 8 + size] + crc = struct.unpack_from('>I', payload, pos + 8 + size)[0] + self.assertEqual(crc, zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (width, height, 8, 2, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + chunks.append(kind) + pos += size + 12 + self.assertEqual(pos, len(payload)) + self.assertEqual(chunks[0], b'IHDR') + self.assertEqual(chunks[-1], b'IEND') + self.assertIn(b'npAD', chunks) + rows = zlib.decompress(compressed) + self.assertEqual(len(rows), height * (width * 3 + 1)) + self.assertTrue(all(rows[row * (width * 3 + 1)] == 0 for row in range(height))) + + def test_seeded_png_has_correct_crc_pixels_and_exact_size(self): + first = self.module.make_png(32, 24, 4096, 123) + self.assertEqual(len(first), 4096) + self.assertEqual(first, self.module.make_png(32, 24, 4096, 123)) + self.assertNotEqual(first, self.module.make_png(32, 24, 4096, 124)) + self.inspect_png(first, 32, 24) + + def test_both_experiment_payloads_decode_and_reach_exact_limits(self): + for name, width, height, size, seed in self.module.CASES: + with self.subTest(name=name): + payload = self.module.make_png(width, height, size, seed) + self.assertEqual(len(payload), size) + self.assertLessEqual(len(payload), 10 * 1024 * 1024) + self.assertLessEqual(width * height, 40_000_000) + self.inspect_png(payload, width, height) + + def test_impossible_png_size_is_rejected(self): + with self.assertRaises(ValueError): + self.module.make_png(32, 32, 64, 1) + + def test_docker_memory_uses_binary_and_decimal_units(self): + for raw, expected in [('1.5MiB / 512MiB', 1572864), ('2MB / 512MB', 2000000), ('4096B / 1GiB', 4096)]: + self.assertEqual(self.module.memory_bytes(raw), expected) + with self.assertRaises(ValueError): + self.module.memory_bytes('not available') + + def test_memory_summary_separates_baseline_and_load_for_each_replica(self): + samples = [ + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 10}, + {'container_id': 'b', 'phase': 'baseline', 'used_bytes': 20}, + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 12}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 35}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 30}, + {'container_id': 'b', 'phase': 'load', 'used_bytes': 50}, + ] + result = self.module.summarize_memory(samples, ['a', 'b']) + self.assertEqual(result['a']['baseline_bytes'], 12) + self.assertEqual(result['a']['sampled_peak_bytes'], 35) + self.assertEqual(result['a']['delta_bytes'], 23) + self.assertEqual(result['b']['sampled_peak_bytes'], 50) + with self.assertRaises(RuntimeError): + self.module.summarize_memory(samples, ['a', 'b', 'missing']) + + def test_latency_percentiles_and_overlap_are_explicit(self): + self.assertEqual(self.module.latency_summary(list(range(1, 11))), + {'p50_ms': 5.5, 'p95_ms': 10, 'max_ms': 10}) + self.assertEqual(self.module.max_inflight([(0, 4), (1, 3), (2, 5)]), 3) + self.assertEqual(self.module.max_inflight([(0, 1), (1, 2)]), 1) + + def test_docker_stream_ignores_ansi_only_lines_between_samples(self): + cid = 'a' * 64 + row = json.dumps({'ID': cid, 'Name': 'lab4-api-1', 'MemUsage': '12MiB / 512MiB'}) + stats = self.module.DockerStats([cid]) + stats.process = SimpleNamespace(stdout=iter(['\x1b[H' + row + '\n', ' \x1b[K\n', row + '\n'])) + stats.mark('small', 'baseline') + stats._read() + self.assertIsNone(stats.error) + self.assertEqual(len(stats.samples), 2) + + def test_rgba64_profile_has_valid_header_crc_and_zero_scanlines(self): + payload = self.module.make_rgba64_png(8, 5) + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, data = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (8, 5, 16, 6, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(5 * (1 + 8 * 8))) + self.assertEqual(pos, len(payload)) + + def test_max_pixels_profile_fits_byte_limit_without_full_raw_buffer(self): + payload = self.module.make_rgba64_png(8000, 5000) + self.assertLess(len(payload), 10 * 1024 * 1024) + self.assertEqual(struct.unpack_from('>IIBB', payload, 16), (8000, 5000, 16, 6)) + + def test_upload_uses_requested_jpeg_mime(self): + captured = [] + class Opener: + def open(self, request, timeout): + captured.append(request) + response = io.BytesIO(b'') + response.code, response.headers = 415, {} + return response + client = SimpleNamespace(base='http://localhost', opener=Opener(), timeout=1) + result = self.module.upload(None, client, {'token': 'synthetic'}, {'id': 'chat'}, b'jpeg', 'hash', + threading.Barrier(1), 0, 'decoder', mime='image/jpeg') + self.assertEqual(captured[0].get_header('Content-type'), 'image/jpeg') + self.assertTrue(captured[0].full_url.endswith('decoder-0.jpg')) + self.assertEqual(result['status'], 415) + + def test_upload_counts_the_queued_snapshot_as_accepted(self): + attachment = {'id': 'a1', 'sha256': 'hash', 'size_bytes': 3, 'status': 'queued', + 'owner_id': 'user', 'chat_id': 'chat'} + response = io.BytesIO(json.dumps(attachment).encode()) + response.code, response.headers = 202, {} + client = SimpleNamespace(base='http://localhost', opener=Mock(), timeout=1) + client.opener.open.return_value = response + result = self.module.upload(Mock(), client, {'id': 'user', 'token': 'synthetic'}, + {'id': 'chat'}, b'png', 'hash', threading.Barrier(1), 0, 'small') + self.assertTrue(result['ok'], result) + self.assertEqual(result['attachment_id'], 'a1') + + def test_sample_download_waits_for_publication(self): + client = Mock() + headers = Message() + headers['Content-Type'] = 'image/jpeg' + client.call.return_value = {'bytes': b'jpeg', 'headers': headers, 'status': 200} + result = self.module.verify_download(client, 'reader', 'a1', b'jpeg', 'image/jpeg') + self.assertEqual([call[0] for call in client.method_calls], ['wait_ready', 'call']) + client.wait_ready.assert_called_once_with('reader', {'id': 'a1'}) + client.call.assert_called_once_with('GET', '/api/v1/attachments/a1/content', actor='reader') + self.assertTrue(result['ok']) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/lab4/upload-memory.py b/scripts/lab4/upload-memory.py new file mode 100644 index 00000000..55da908b --- /dev/null +++ b/scripts/lab4/upload-memory.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +"""Concurrent upload experiment; stdlib only, no stand lifecycle operations.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import re +import statistics +import struct +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from urllib.parse import urlsplit +import zlib + + +ROOT = Path(__file__).resolve().parents[2] +sys.dont_write_bytecode = True + +CASES = ( + ('small', 512, 512, 1024 * 1024, 41), + ('limit', 1600, 1600, 10 * 1024 * 1024, 42), +) +MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None) + + +def png_chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def make_png(width, height, size, seed): + rng = random.Random(seed) + rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1)) + ending = png_chunk(b'IEND', b'') + gap = size - len(prefix) - len(ending) + if gap < 0 or 0 < gap < 12: + raise ValueError('target cannot contain this PNG') + # Ancillary, private, reserved bit valid: unknown npAD is safely ignored. + padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b'' + return prefix + padding + ending + + +def make_rgba64_png(width, height): + encoder = zlib.compressobj(1) + row = bytes(1 + width * 8) + pieces = [encoder.compress(row) for _ in range(height)] + pieces.append(encoder.flush()) + header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + + png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b'')) + + +def payload_for(case): + name, width, height, size, seed = case + return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed) + + +def memory_bytes(value): + match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip()) + if not match: + raise ValueError('unrecognized docker memory value') + units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2, + 'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4} + return round(float(match[1]) * units[match[2]]) + + +def summarize_memory(samples, container_ids): + result = {} + for container_id in container_ids: + baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline'] + load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load'] + if not baseline or not load: + raise RuntimeError('missing baseline or load sample for an API replica') + before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load) + result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before, + 'baseline_samples': len(baseline), 'load_samples': len(load)} + return result + + +def latency_summary(values): + ordered = sorted(values) + return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1], + 'max_ms': ordered[-1]} + + +def max_inflight(intervals): + active, peak = 0, 0 + events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals] + for _, delta in sorted(events): + active += delta + peak = max(peak, active) + return peak + + +class DockerStats: + def __init__(self, container_ids): + self.ids = container_ids + self.samples = [] + self.lock = threading.Condition() + self.case, self.phase = '', 'idle' + self.error = None + self.started = time.monotonic() + + def __enter__(self): + self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, encoding='utf-8', errors='replace', bufsize=1) + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + return self + + def _read(self): + for line in self.process.stdout: + try: + cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + if not cleaned: + continue + raw = json.loads(cleaned) + reported_id = raw.get('ID') or raw.get('Container', '') + container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid)) + used = memory_bytes(raw['MemUsage']) + limit = memory_bytes(raw['MemUsage'].split('/')[1]) + if used == 0: + continue + with self.lock: + self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used, + 'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6), + 'case': self.case, 'phase': self.phase}) + self.lock.notify_all() + except (ValueError, KeyError, IndexError, StopIteration) as exc: + with self.lock: + self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120]) + self.lock.notify_all() + with self.lock: + self.lock.notify_all() + + def mark(self, case, phase): + with self.lock: + self.case, self.phase = case, phase + + def snapshot(self, case): + with self.lock: + return [dict(s) for s in self.samples if s['case'] == case] + + def wait_for_baseline(self, case, timeout=20): + deadline = time.monotonic() + timeout + with self.lock: + while True: + seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'} + if seen == set(self.ids): + return + if self.error or self.process.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError('docker stats baseline unavailable') + self.lock.wait(min(.1, max(0, deadline - time.monotonic()))) + + def __exit__(self, *_): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + self.thread.join(timeout=5) + self.process.stdout.close() + + +def api_containers(args): + command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)] + if args.project_name: + command.extend(['--project-name', args.project_name]) + result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + ids = result.stdout.split() + if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids): + raise RuntimeError('compose has no running API containers') + return ids + + +def container_states(container_ids): + template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},' + '"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},' + '"exit_code":{{json .State.ExitCode}}}') + result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())} + + +def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'): + result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)} + barrier.wait(timeout=30) + started = time.monotonic() + try: + suffix = '.jpg' if mime == 'image/jpeg' else '.png' + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix + req = urllib.request.Request(client.base + path, data=content, method='POST', headers={ + 'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime, + 'Content-Length': str(len(content)), 'Accept': 'application/json'}) + try: + response = client.opener.open(req, timeout=client.timeout) + except urllib.error.HTTPError as exc: + response = exc + with response: + result['status'] = response.code + result['instance_id'] = response.headers.get('X-Instance-Id') + body = response.read(65537) + if result['status'] != 202: + result['error'] = 'http_error' + elif len(body) > 65536: + result['error'] = 'oversized_metadata' + else: + attachment = json.loads(body) + client_module.shape(attachment, 'Attachment') + valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content) + and attachment['status'] == 'queued' and attachment['owner_id'] == actor['id'] + and attachment['chat_id'] == chat['id']) + result['ok'] = valid + result['attachment_id'] = attachment['id'] + if not valid: + result['error'] = 'attachment_metadata_mismatch' + except Exception as exc: + # Exception text and response bodies are deliberately excluded from evidence. + result['error'] = type(exc).__name__ + finished = time.monotonic() + result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3)) + return result + + +def verify_download(client, reader, attachment_id, content, mime): + # Poll after the upload measurements so latency ends at the durable handoff. + client.wait_ready(reader, {'id': attachment_id}) + response = client.call('GET', '/api/v1/attachments/' + attachment_id + '/content', actor=reader) + body = response['bytes'] + return {'ok': body == content and response['headers'].get_content_type() == mime, + 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest(), 'status': response['status']} + + +def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case): + name, width, height, size, seed = case + content = payload_for(case) + size = len(content) + checksum = hashlib.sha256(content).hexdigest() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + state_before = container_states(stats.ids) + stats.mark(name, 'baseline') + stats.wait_for_baseline(name) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name) + for index, client in enumerate(clients)] + stats.mark(name, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + duration = time.monotonic() - started + # Retain a tail because docker stats reports its own sampled windows. + time.sleep(args.tail_seconds) + stats.mark(name, 'idle') + samples = stats.snapshot(name) + memory_error = None + try: + memory = summarize_memory(samples, stats.ids) + except RuntimeError as exc: + memory, memory_error = {}, str(exc) + intervals = [(r['_started'], r['_finished']) for r in results] + peak_requests = max_inflight(intervals) + successes = [r for r in results if r['ok']] + download = {'ok': False} + if successes: + try: + download = verify_download(bootstrap_client, reader, successes[0]['attachment_id'], content, 'image/png') + except Exception as exc: + download['error'] = type(exc).__name__ + distinct = len({r['attachment_id'] for r in successes}) + state_after = container_states(stats.ids) + stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed'] + and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum, + 'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2, + 'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4), + 'requests': args.concurrency, 'attempted_bytes': size * args.concurrency, + 'successful_bytes': size * len(successes), 'accepted': len(successes), + 'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)), + 'errors': len(results) - len(successes), 'duration_s': round(duration, 6), + 'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct, + 'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory, + 'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download, + 'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable, + 'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10 + and download['ok'] and not memory_error and stable)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml') + parser.add_argument('--project-name', default=None) + parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here') + parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard', + help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)') + parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker') + args = parser.parse_args() + selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,) + if args.check_generator: + print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload), + 'sha256': hashlib.sha256(payload).hexdigest()} + for name, width, height, size, seed in selected_cases + for payload in [payload_for((name, width, height, size, seed))]], indent=2)) + return 0 + if not args.output: + parser.error('--output is required for live execution') + if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s') + target = urlsplit(args.base_url) + if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment: + parser.error('--base-url must be HTTP(S) without credentials, query or fragment') + report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False, + 'driver': {'os': platform.system(), 'architecture': platform.machine(), + 'logical_cpus': os.cpu_count(), 'python': platform.python_version()}, + 'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second', + 'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail', + 'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds, + 'same_bytes_per_case': True, 'p95_method': 'nearest rank', + 'profile': args.profile, + 'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard' + else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []} + try: + sys.path.insert(0, str(ROOT / 'tests')) + import client as client_module + ids = api_containers(args) + report['api_container_ids'] = ids + bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout) + report['stage'] = 'bootstrap' + actor, reader, _, chat = bootstrap_client.bootstrap() + with DockerStats(ids) as stats: + for case in selected_cases: + report['stage'] = case[0] + report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case)) + if stats.error: + report['sampling_error'] = stats.error + report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error + report['stage'] = 'complete' + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)): + # Public helper diagnostics contain only paths/schema names, never values or credentials. + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'output': str(args.output), + 'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'], + 'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'], + 'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']], + 'error': report.get('error')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/lab5/track2.py b/scripts/lab5/track2.py new file mode 100644 index 00000000..d00b10b8 --- /dev/null +++ b/scripts/lab5/track2.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Run only on the disposable lab5 fault-overlay Compose stand. Retains volumes.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import time +from urllib.parse import urlsplit +import uuid +import zlib + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + +MARKERS = "/tmp/lab5-faults/" +VARIANTS = {"thumbnail", "crop", "optimized"} + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def fixture(seed): + """A deterministic textured PNG produces different WebP bytes at the two qualities.""" + size = 160 + rows = bytearray() + for y in range(size): + rows.append(0) + for x in range(size): + rows.extend(((x * 3 + y * 7 + seed) % 256, (x * y + seed) % 256, (x ^ y ^ seed) % 256)) + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"")) + + +def command(args, *, allow_failure=False, binary=False, timeout=60): + proc = subprocess.run(args, cwd=ROOT, capture_output=True, timeout=timeout) + if proc.returncode and not allow_failure: + raise Failure(f"{args[:4]} exited {proc.returncode}: " + proc.stderr.decode(errors="replace")[-600:]) + if allow_failure: + return proc + return proc.stdout if binary else proc.stdout.decode().strip() + + +def wait(test, label, timeout=90): + end = time.monotonic() + timeout + while time.monotonic() < end: + value = test() + if value: + return value + time.sleep(0.16) + raise Failure("timed out waiting for " + label) + + +class Experiment: + def __init__(self, base_url, project): + self.compose = ["docker", "compose", "-p", project, "-f", "compose.yaml", "-f", "compose.faults.yaml"] + address = urlsplit(base_url) + port = self.dc("port", "lb", "8080").splitlines()[0].rsplit(":", 1)[-1] + check(address.hostname in {"127.0.0.1", "localhost"} and str(address.port) == port, + "base URL must match the explicit local Compose stand") + self.worker = self.ids("worker")[0] + self.s3 = self.ids("s3")[0] + self.bucket = self.exe(self.worker, "printenv", "S3_BUCKET") + self.apis = self.ids("api") + check(len(self.apis) >= 2, "two API replicas required") + check("--barriers=" + MARKERS.rstrip("/") in command(["docker", "inspect", "-f", "{{json .Config.Cmd}}", self.worker]), + "worker must run with compose.faults.yaml") + for api in self.apis: + check(self.exe(api, "printenv", "LAB5_BARRIERS") == MARKERS.rstrip("/"), "API barrier overlay required") + self.client = Client(base_url) + self.actor, self.reader, _, self.chat = self.client.bootstrap() + self.content = fixture(41) + self.stopped = set() + self.armed = set() + + def dc(self, *args, **kwargs): + return command(self.compose + list(args), **kwargs) + + def ids(self, service): + ids = self.dc("ps", "-q", service).splitlines() + check(bool(ids), service + " must be running") + return ids + + def exe(self, container, *args, **kwargs): + return command(["docker", "exec", container, *args], **kwargs) + + def sql(self, statement): + return self.dc("exec", "-T", "postgres", "psql", "-XqAt", "-v", "ON_ERROR_STOP=1", + "-U", "messenger", "-d", "messenger", "-c", statement) + + def marker(self, container, stem, operation): + path = MARKERS + stem + if operation == "arm": + self.exe(container, "mkdir", "-p", MARKERS) + self.armed.add((container, stem)) + self.exe(container, "touch", path + ".arm") + elif operation == "hit": + return self.exe(container, "test", "-f", path + ".hit", allow_failure=True).returncode == 0 + elif operation == "release": + self.exe(container, "rm", "-f", path + ".arm", path + ".hit", path + ".id") + self.armed.discard((container, stem)) + + def arm(self, stem, containers): + for container in containers: + self.marker(container, stem, "arm") + + def hit(self, stem, containers): + return wait(lambda: next((c for c in containers if self.marker(c, stem, "hit")), None), stem) + + def stop(self, service): + self.stopped.add(service) + self.dc("stop", "-t", "3", service) + + def start(self, service): + self.dc("start", service) + self.stopped.discard(service) + + def crash(self, container): + command(["docker", "kill", "--signal", "KILL", container]) + command(["docker", "start", container]) + + def cli(self, *args, **kwargs): + return self.exe(self.worker, "/usr/local/bin/worker", *args, **kwargs) + + def object_hash(self,key): + # Read through the storage process's loopback-only filer endpoint. This + # can observe a worker's PUT before HTTP publication has committed. + url = "http://127.0.0.1:8888/buckets/" + self.bucket + "/" + key + fetched = self.exe(self.s3,"wget","-SO-",url,allow_failure=True) + if fetched.returncode: + check(b"404 Not Found" in fetched.stderr, "cannot inspect filer object " + key) + return None + return digest(fetched.stdout) + + def upload(self, data=None): + data = self.content if data is None else data + att = self.client.call("POST", "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=track2.png", 202, + actor=self.actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment")["json"] + check(att["status"] == "queued" and att["sha256"] == digest(data), "invalid accepted upload") + return str(uuid.UUID(att["id"])) + + def state(self, aid): + return json.loads(self.sql(f""" + SELECT json_build_object('target',a.target_version,'published',a.published_version, + 'jobs',(SELECT json_agg(json_build_object('version',j.pipeline_version,'id',j.job_id, + 'attempt',j.attempt,'state',j.state,'error',j.error_code,'confirmed',j.published_at IS NOT NULL) + ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id), + 'variant_count',(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id)) + FROM attachments a WHERE a.id='{aid}'""")) + + def result(self, aid, data=None): + data = self.content if data is None else data + att = self.client.wait_ready(self.reader, {"id": aid}, 120) + base = "/api/v1/attachments/" + aid + "/content" + original = self.client.call("GET", base, actor=self.reader)["bytes"] + check(original == data, "accepted original changed") + hashes = {} + for variant in att["variants"]: + name = variant["name"] + obj = self.client.call("GET", base + "?variant=" + name, actor=self.reader) + h = digest(obj["bytes"]) + check(h == variant["sha256"] and len(obj["bytes"]) == variant["size_bytes"], "variant bytes differ from metadata") + check(name not in hashes, "duplicate public variant") + hashes[name] = h + check(set(hashes) == VARIANTS, "missing public variant") + state = self.state(aid) + rows = self.sql(f"""SELECT variant||' '||encode(sha256,'hex') FROM attachment_variants + WHERE attachment_id='{aid}' AND pipeline_version={int(state['published'])}""") + check(dict(row.split() for row in rows.splitlines()) == hashes, "SQL and downloaded bytes differ") + return {"attachment": aid, "original_sha256": digest(original), "variant_sha256": hashes, **state} + + def queued(self): + self.stop("broker") + aid = self.upload() + state = self.state(aid) + check(state["jobs"][0]["state"] == "queued" and not state["jobs"][0]["confirmed"], "expected pending outbox") + self.client.call("GET", "/api/v1/attachments/" + aid + "/content", 409, actor=self.reader) + return aid + + def staged_crash(self): + data = fixture(uuid.uuid4().int % 255) + marker = digest(data) + "-staged" + self.arm(marker, self.apis) + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(self.upload, data) + owner = self.hit(marker, self.apis) + path = MARKERS + marker + ".id" + def staged_id(): + p = self.exe(owner, "cat", path, allow_failure=True) + return p.stdout.decode().strip() if p.returncode == 0 else None + aid = str(uuid.UUID(wait(staged_id, "uncommitted staging UUID"))) + self.crash(owner) + try: + future.result() + except Failure: + pass + else: + raise Failure("interrupted staging unexpectedly returned 202") + for api in self.apis: + self.marker(api, marker, "release") + rows = int(self.sql(f"SELECT count(*) FROM attachments WHERE id='{aid}'")) + check(rows == 0, "staged-only upload acquired a logical attachment") + check(self.object_hash("staging/" + aid) == digest(data), "crashed upload did not leave the expected staged bytes") + return {"unaccepted": aid, "source_sha256": digest(data), "attachment_rows": rows} + + def outbox_restart(self): + aid = self.queued() + message = self.client.message(self.actor, self.chat, "Принято без брокера", attachments=[aid])["json"] + before = self.state(aid) + self.dc("restart", "api") + def api_ready(): + try: + return self.client.call("GET", "/health/ready", schema="Health")["status"] == 200 + except Failure: + return False + wait(api_ready, "API readiness without broker") + check(self.state(aid) == before, "durable intent changed during API restart") + self.start("broker") + result = self.result(aid) + check(result["jobs"][0]["id"] == before["jobs"][0]["id"], "restart recreated job") + path = "/api/v1/chats/" + self.chat["id"] + "/messages" + history = self.client.call("GET", path, actor=self.reader, schema="MessagePage")["json"]["items"] + check(message in history, "confirmed message disappeared") + return {"before": before, "message": message["id"], "after": result} + + def published_crash(self): + aid = self.queued() + self.stop("worker") + marker = aid + "-v1-published" + self.arm(marker, self.apis) + self.start("broker") + owner = self.hit(marker, self.apis) + for api in self.apis: + if api != owner: + self.marker(api, marker, "release") + before = self.state(aid) + check(not before["jobs"][0]["confirmed"], "SQL confirmation already committed") + # Attaching a queued file must finish even while the publisher holds its row lock. + linked = self.client.message(self.actor, self.chat, "", attachments=[aid]) + check(linked["json"]["attachment_ids"] == [aid], "message lost its attachment") + self.crash(owner) + self.marker(owner, marker, "release") + wait(lambda: self.state(aid)["jobs"][0]["confirmed"], "republished envelope") + def queued_copies(): + rows = json.loads(self.dc("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", + "name", "messages_ready", "--formatter=json")) + copies = next((row["messages_ready"] for row in rows if row["name"] == "attachment.jobs"), 0) + return copies if copies >= 2 else 0 + copies = wait(queued_copies, "two confirmed broker copies while consumers are stopped") + confirmed = self.state(aid) + self.dc("restart", "broker") + def after_restart(): + try: + return queued_copies() + except Failure: + return 0 + restored = wait(after_restart, "durable queue after broker restart", 120) + check(copies == restored and self.state(aid) == confirmed, "confirmed queue or SQL job changed across broker restart") + self.start("worker") + result = self.result(aid) + check(result["variant_count"] == 3 and result["jobs"][0]["attempt"] == 1, "publication crash duplicated the effect") + return {"broker_confirm_before_sql_commit": before, "confirmed_job_before_broker_restart": confirmed, + "message_while_publisher_held": {"id": linked["json"]["id"], "status": linked["status"], + "seconds": round(linked["elapsed"], 3)}, + "broker_messages_before_restart": copies, "broker_messages_after_restart": restored, "after": result} + + def worker_crash(self, phase, reference): + aid = self.queued() + marker = aid + "-v1-" + phase + self.arm(marker, [self.worker]) + self.start("broker") + self.hit(marker, [self.worker]) + before = self.state(aid) + check(before["variant_count"] == (3 if phase == "committed" else 0), "crash occurred on the wrong side of SQL commit") + observed = {name: self.object_hash(f"variants/{aid}/1/{name}/{value}") for name,value in reference.items()} + expected = {name:None for name in VARIANTS} if phase == "claimed" else reference + check(observed == expected, "crash occurred on the wrong side of storage writes") + if phase == "committed": + duplicate = aid + "-v1-duplicate" + self.arm(duplicate, [self.worker]) + self.crash(self.worker) + self.marker(self.worker, marker, "release") + if phase == "committed": + self.hit(duplicate, [self.worker]) + self.marker(self.worker, duplicate, "release") + result = self.result(aid) + want = 1 if phase == "committed" else 2 + check(result["jobs"][0]["attempt"] == want and result["variant_count"] == 3, "incorrect crash recovery") + return {"phase": phase, "before": before, "storage_sha256_at_barrier": observed, "after": result} + + def duplicate_ack(self, aid, version): + prefix = f"{aid}-v{version}-" + for phase in ["duplicate", "acked"]: + self.arm(prefix + phase, [self.worker]) + self.cli("duplicate", aid, str(version), "1") + for phase in ["duplicate", "acked"]: + self.hit(prefix + phase, [self.worker]) + self.marker(self.worker, prefix + phase, "release") + + def lease_takeover(self): + aid = self.queued() + objects = aid + "-v1-objects" + committed = aid + "-v1-committed" + acked = aid + "-v1-acked" + self.arm(objects, [self.worker]) + self.arm(committed, [self.worker]) + self.start("broker") + self.hit(objects, [self.worker]) + first = self.state(aid) + self.hit(committed, [self.worker]) + winner = self.result(aid) + check(first["jobs"][0]["attempt"] == 1 and winner["jobs"][0]["attempt"] == 2 + and first["jobs"][0]["id"] == winner["jobs"][0]["id"], "lease takeover changed logical work") + for barrier in [committed, objects]: + self.arm(acked, [self.worker]) + self.marker(self.worker, barrier, "release") + self.hit(acked, [self.worker]) + self.marker(self.worker, acked, "release") + after = self.result(aid) + check(after == winner and after["variant_count"] == 3, "expired worker modified published result") + return {"first_held": first, "second_committed": winner, "after_first_ack": after} + + def reordered(self, v1_hashes): + aid = self.queued() + marker = aid + "-v1-objects" + self.arm(marker, [self.worker]) + self.start("broker") + self.hit(marker, [self.worker]) + self.duplicate_ack(aid, 1) + self.cli("reprocess", aid, "2") + before = self.result(aid) + check(before["published"] == 2 and before["variant_count"] == 3 and before["jobs"][0]["state"] == "superseded" + and before["jobs"][0]["attempt"] == 1, + "new version did not finish while first worker held its result") + check(v1_hashes["optimized"] != before["variant_sha256"]["optimized"], "fixture must distinguish the profiles") + acked = aid + "-v1-acked" + self.arm(acked, [self.worker]) + self.marker(self.worker, marker, "release") + self.hit(acked, [self.worker]) + self.marker(self.worker, acked, "release") + for version in [2, 1]: + self.duplicate_ack(aid, version) + after = self.result(aid) + check(after == before, "stale commit or delivery changed new result") + return {"reference_v1_webp": v1_hashes["optimized"], "before_releasing_v1": before, + "after_acks_in_order": [2, 1], "after": after} + + def retry_replay(self): + aid = self.queued() + real = "staging/" + aid + self.sql(f"UPDATE attachment_jobs SET source_key='staging/{uuid.uuid4()}' WHERE attachment_id='{aid}'") + self.start("broker") + wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "retry exhaustion", 180) + failed = self.state(aid) + limit = int(self.exe(self.worker, "printenv", "JOB_MAX_ATTEMPTS")) + job = failed["jobs"][0] + check(job["attempt"] == limit and job["error"] == "source_missing", "wrong failure budget") + entries = json.loads(self.cli("failed")) + check(any(row["job_id"] == job["id"] for row in entries), "missing failed job") + self.sql(f"UPDATE attachment_jobs SET source_key='{real}' WHERE attachment_id='{aid}'") + self.cli("replay", aid, "track2") + result = self.result(aid) + check(result["jobs"][0]["attempt"] == limit + 1, "replay reused a stale attempt") + rejected = self.cli("replay", aid, "track2", allow_failure=True) + check(rejected.returncode != 0, "accepted replay of successful job") + count = int(self.sql(f"SELECT count(*) FROM attachment_replays WHERE job_id='{job['id']}' AND actor='track2'")) + check(count == 1, "incorrect audit count") + return {"injection": "absent source key assigned only to synthetic job", "exhausted": failed, "after": result, "audit_rows": count} + + def permanent_failure(self, other_sha): + self.stop("broker") + aid = self.upload(fixture(19)) + self.sql(f"UPDATE attachment_jobs SET source_key='blobs/{other_sha}' WHERE attachment_id='{aid}'") + self.start("broker") + wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "permanent source mismatch") + failed = self.state(aid) + j = failed["jobs"][0] + check(j["attempt"] == 1 and j["error"] == "source_mismatch" and failed["variant_count"] == 0, + "permanent failure consumed additional attempts") + self.sql(f"UPDATE attachment_jobs SET source_key='staging/{aid}' WHERE attachment_id='{aid}'") + return {"injection": "different valid original assigned only to synthetic job", "failed": failed} + + def cleanup_stage(self, aid, created): + # Run normal GC after its real minimum grace. No object timestamps are forged. + remaining = created + 125 - time.monotonic() + if remaining > 0: + time.sleep(remaining) + def run_gc(apply): + args = ["run", "--rm", "--no-deps", "-T", "reconcile", "--grace=2m"] + if apply: + args.append("--apply") + return [json.loads(line) for line in self.dc(*args, timeout=180).splitlines()] + def matches(rows): + return [row for row in rows if row.get("key") == "staging/" + aid] + before = matches(run_gc(False)) + check(len(before) == 1 and before[0]["kind"] == "stale_staging", "unaccepted stage not reported") + applied = matches(run_gc(True)) + check(len(applied) == 1 and applied[0]["applied"], "unaccepted stage not removed") + check(not matches(run_gc(False)), "removed stage still listed") + return {"before": before[0], "applied": applied[0], "after_findings": 0} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True) + # Match the project selected for the Compose startup, including isolated review runs. + parser.add_argument("--project", choices=[os.environ.get("COMPOSE_PROJECT_NAME") or "lab5"], required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + exp = Experiment(args.base_url, args.project) + started = time.monotonic() + report = {"project": args.project, "complete": False, "scenarios": {}, "seconds": {}, + "started_at":datetime.now(timezone.utc).isoformat()} + args.output.parent.mkdir(parents=True, exist_ok=True) + def save(): + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + def record(name, callback): + print(name, flush=True) + beginning = time.monotonic() + report["scenarios"][name] = callback() + report["seconds"][name] = round(time.monotonic() - beginning, 3) + save() + return report["scenarios"][name] + save() + try: + stage = record("staging_before_commit", exp.staged_crash) + created = time.monotonic() + accepted = record("commit_before_publish", exp.outbox_restart)["after"] + record("broker_confirm_before_sql", exp.published_crash) + for phase in ["claimed", "objects", "committed"]: + record("worker_crash_" + phase, lambda: exp.worker_crash(phase, accepted["variant_sha256"])) + record("same_version_lease_takeover", exp.lease_takeover) + record("parallel_duplicate_and_reorder", lambda: exp.reordered(accepted["variant_sha256"])) + record("retry_exhaustion_replay", exp.retry_replay) + record("permanent_failure", lambda: exp.permanent_failure(accepted["original_sha256"])) + record("unaccepted_staging_cleanup", lambda: exp.cleanup_stage(stage["unaccepted"], created)) + report["complete"] = True + report["finished_at"] = datetime.now(timezone.utc).isoformat() + report["elapsed_seconds"] = round(time.monotonic() - started, 3) + save() + finally: + for service in sorted(exp.stopped): + exp.start(service) + for container, stem in list(exp.armed): + exp.marker(container, stem, "release") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..5f86f138 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/restart.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0restart.sh" +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..0a715fde --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Restart hook of `make test-persistence`: stops the whole stand gracefully and +# starts the same containers again; volumes are kept. +set -eu +cd "$(dirname "$0")/.." +docker compose stop +# Compose starts both one-shot jobs and waits for their successful exit +# before either API replica or the worker can start. +docker compose up -d --wait +sh scripts/wait-ready.sh diff --git a/scripts/rollout.sh b/scripts/rollout.sh new file mode 100755 index 00000000..4710e64c --- /dev/null +++ b/scripts/rollout.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout, +# MIT): new replicas start next to the old ones, and the old ones go once all +# new ones are healthy; if one is not, the new ones are removed instead. +# +# With a stopped replica docker rollout would replace only that one and leave +# two versions running, so the script refuses. The migrate container is +# recreated from the new image without starting: the `up` of docker rollout +# then applies the new migrations once, before any new replica starts. +set -eu +cd "$(dirname "$0")/.." + +idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api) +if [ -n "$idle" ]; then + echo "some API replicas are not running; start them first: docker compose start api" >&2 + exit 2 +fi + +version=v0.14 +checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b +tool=bin/docker-rollout + +if ! printf '%s %s\n' "$checksum" "$tool" | sha256sum -c - >/dev/null 2>&1; then + mkdir -p bin + curl -fsSL -o "$tool.download" "https://github.com/wowu/docker-rollout/releases/download/$version/docker-rollout" + if ! printf '%s %s\n' "$checksum" "$tool.download" | sha256sum -c - >/dev/null 2>&1; then + rm -f "$tool.download" + echo "docker-rollout $version: checksum mismatch, not used" >&2 + exit 1 + fi + mv "$tool.download" "$tool" +fi + +docker compose build api +docker compose up --no-start --no-deps --force-recreate migrate +# Run through sh: Windows cannot use the script as a docker CLI plugin. +if ! sh "$tool" rollout --wait-after-healthy 4 api; then + # A failed migration leaves the new containers created but not started. + created=$(docker compose ps -aq --status created api) + if [ -n "$created" ]; then + docker rm $created >/dev/null + fi + exit 1 +fi diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..76322518 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows entry point of scripts/stop-one.sh: tests/resilience.py starts the +rem hook without a shell, and Windows cannot execute a .sh file directly. +sh "%~dp0stop-one.sh" +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..6971e1b4 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Failover hook of `make test-failover`: gracefully stops the one running API +# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else. +# Restore it with: docker compose start api +set -eu +cd "$(dirname "$0")/.." + +target=${TARGET_INSTANCE_ID:-} +case "$target" in +'' | *[!0-9a-f]*) + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 + ;; +esac +if [ ${#target} -ne 16 ]; then + echo "TARGET_INSTANCE_ID must be the 16 lowercase hex characters of an X-Instance-Id" >&2 + exit 2 +fi + +matches=$(sh scripts/api-instances.sh | awk -v t="$target" '$3 == "running" && $4 == t { print $1 }') +count=$(printf '%s\n' "$matches" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo "expected one running API replica with X-Instance-Id $target, found $count" >&2 + exit 2 +fi +docker stop "$matches" >/dev/null +echo "stopped API container $matches (X-Instance-Id $target)" diff --git a/scripts/wait-ready.sh b/scripts/wait-ready.sh new file mode 100644 index 00000000..70f93502 --- /dev/null +++ b/scripts/wait-ready.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Compose may retain the balancer's earlier healthy status while API backends +# are recreated. Wait for fresh successful responses through the real LB. +set -eu +cd "$(dirname "$0")/.." +attempt=0 +stable=0 +while [ "$attempt" -lt 60 ]; do + if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then + stable=$((stable + 1)) + if [ "$stable" -ge 3 ]; then + echo "balancer readiness confirmed" + exit 0 + fi + else + stable=0 + fi + attempt=$((attempt + 1)) + sleep 1 +done +echo "balancer did not become ready within 60 probes" >&2 +exit 1 diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go new file mode 100644 index 00000000..9b583e0d --- /dev/null +++ b/tools/concurrency/client.go @@ -0,0 +1,135 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" +) + +// client is a minimal typed HTTP client for the messenger API. +type client struct { + base string + http *http.Client +} + +func newClient(baseURL string) *client { + return &client{base: baseURL, http: &http.Client{Timeout: 10 * time.Second}} +} + +func (c *client) post(path, token string, body, out any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+path, bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req, token, http.StatusCreated, out) +} + +// postMessage sends one message under a fresh Idempotency-Key. +func (c *client) postMessage(chatID, token, text string, out any) error { + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Idempotency-Key", uuid.NewString()) + return c.do(req, token, http.StatusCreated, out) +} + +// sendWithKey sends one attempt on a connection of its own, so that the +// attempts of one key reach different replicas. +func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { + var msg chat.MessageResponse + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return 0, msg, "", err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return 0, msg, "", err + } + req.Close = true + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Idempotency-Key", key) + resp, err := c.http.Do(req) + if err != nil { + return 0, msg, "", err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return 0, msg, "", err + } + if resp.StatusCode == http.StatusCreated { + if err := json.Unmarshal(payload, &msg); err != nil { + return 0, msg, "", err + } + } + return resp.StatusCode, msg, resp.Header.Get("X-Instance-Id"), nil +} + +// login opens a session with Basic credentials and returns its bearer token. +func (c *client) login(username, password string) (string, error) { + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + req.SetBasicAuth(username, password) + var session struct { + Token string `json:"token"` + } + if err := c.do(req, "", http.StatusCreated, &session); err != nil { + return "", err + } + return session.Token, nil +} + +// get requests the largest page the contract allows, continuing from cursor +// when it is set. +func (c *client) get(path, token, cursor string, out any) error { + query := url.Values{"limit": {"100"}} + if cursor != "" { + query.Set("cursor", cursor) + } + req, err := http.NewRequest(http.MethodGet, c.base+path+"?"+query.Encode(), nil) + if err != nil { + return err + } + return c.do(req, token, http.StatusOK, out) +} + +func (c *client) do(req *http.Request, token string, want int, out any) error { + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return err + } + if resp.StatusCode != want { + return fmt.Errorf("%s %s: status %d, want %d: %s", req.Method, req.URL.Path, resp.StatusCode, want, payload) + } + return json.Unmarshal(payload, out) +} diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go new file mode 100644 index 00000000..a1ac25e6 --- /dev/null +++ b/tools/concurrency/main.go @@ -0,0 +1,322 @@ +// Command concurrency is the black-box race check from the grading rubric. +// +// Phase 1: N clients send M messages each into one chat at the same time, +// then the history is paged and must contain exactly N*M unique messages in +// id order. +// +// Phase 2: in each of K rounds one key is sent P times in parallel through the +// balancer; every attempt must get 201 with one message, at least two replicas +// must answer, the history must hold K messages, and another text must get 409. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/google/uuid" + + "messenger/internal/chat" + "messenger/internal/platform/pagination" + "messenger/internal/user" +) + +// Report is the machine readable outcome saved as evidence. +type Report struct { + BaseURL string `json:"base_url"` + StartedAt time.Time `json:"started_at"` + Clients int `json:"clients"` + PerClient int `json:"per_client"` + Expected int `json:"expected_messages"` + Created int `json:"created_201"` + Failed int `json:"failed_posts"` + Listed int `json:"listed_messages"` + Unique int `json:"unique_ids"` + Missing int `json:"created_but_not_listed"` + Ordered bool `json:"history_ordered_by_id"` + Pages int `json:"pages_read"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + // Idempotency is phase 2. + Idempotency IdempotencyReport `json:"idempotency"` + Passed bool `json:"passed"` +} + +type IdempotencyReport struct { + Rounds int `json:"rounds"` + Parallel int `json:"parallel"` + // SameMessageRounds counts rounds whose attempts all answered 201 with + // one and the same message. + SameMessageRounds int `json:"same_message_rounds"` + // InstancesSeen is the number of distinct X-Instance-Id values among all + // attempts. + InstancesSeen int `json:"instances_seen"` + RequireReplicas bool `json:"require_two_replicas"` + Conflict409 bool `json:"different_body_409"` + History int `json:"history_messages"` + Passed bool `json:"passed"` +} + +func main() { + baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + clients := flag.Int("clients", 10, "number of concurrent clients") + perClient := flag.Int("per-client", 10, "messages sent by each client") + rounds := flag.Int("rounds", 20, "idempotency rounds, one key each") + parallel := flag.Int("parallel", 8, "parallel attempts with the same key per round") + requireReplicas := flag.Bool("require-replicas", true, "fail unless the attempts reached at least two replicas") + output := flag.String("output", "", "optional path for the JSON report") + flag.Parse() + + report, err := run(*baseURL, *clients, *perClient, *rounds, *parallel, *requireReplicas) + if err != nil { + fail(err) + } + text, _ := json.MarshalIndent(report, "", " ") + fmt.Println(string(text)) + if *output != "" { + if err := writeReport(*output, text); err != nil { + fail(err) + } + } + if !report.Passed { + fail(fmt.Errorf("concurrency invariants violated")) + } + fmt.Printf("PASS: %d clients x %d messages produced exactly %d unique ordered messages\n", + report.Clients, report.PerClient, report.Unique) + fmt.Printf("PASS: %d rounds x %d parallel attempts with one key produced %d messages across %d instances\n", + report.Idempotency.Rounds, report.Idempotency.Parallel, report.Idempotency.History, report.Idempotency.InstancesSeen) +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "FAIL:", err) + os.Exit(1) +} + +func writeReport(path string, text []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, append(text, '\n'), 0o644) +} + +func run(baseURL string, clients, perClient, rounds, parallel int, requireReplicas bool) (Report, error) { + if clients < 2 || perClient < 1 || rounds < 1 || parallel < 1 { + return Report{}, fmt.Errorf("clients must be at least 2; per-client, rounds and parallel positive") + } + report := Report{ + BaseURL: baseURL, + StartedAt: time.Now().UTC(), + Clients: clients, + PerClient: perClient, + Expected: clients * perClient, + } + api := newClient(baseURL) + runID := uuid.NewString()[:8] + + members, tokens, err := registerClients(api, runID, clients) + if err != nil { + return report, err + } + room, err := createChat(api, runID, members, tokens[0]) + if err != nil { + return report, err + } + + started := time.Now() + created, failed := sendConcurrently(baseURL, room.ID, tokens, perClient) + report.ElapsedSeconds = time.Since(started).Seconds() + report.Created = len(created) + report.Failed = failed + + listed, pages, err := readHistory(api, room.ID, tokens[0]) + if err != nil { + return report, err + } + report.Pages = pages + report.Listed = len(listed) + report.Unique, report.Missing = compare(created, listed) + report.Ordered = sort.StringsAreSorted(listed) + report.Idempotency, err = checkIdempotency(api, baseURL, runID, members[:2], tokens[0], rounds, parallel, requireReplicas) + if err != nil { + return report, err + } + report.Passed = report.Failed == 0 && + report.Created == report.Expected && + report.Listed == report.Expected && + report.Unique == report.Expected && + report.Missing == 0 && + report.Ordered && + report.Idempotency.Passed + return report, nil +} + +// checkIdempotency runs phase 2 in a chat of its own. +func checkIdempotency(api *client, baseURL, runID string, members []string, token string, rounds, parallel int, requireReplicas bool) (IdempotencyReport, error) { + r := IdempotencyReport{Rounds: rounds, Parallel: parallel, RequireReplicas: requireReplicas} + room, err := createChat(api, runID+"-idem", members, token) + if err != nil { + return r, err + } + instances := map[string]bool{} + var key, text string + for round := range rounds { + key, text = uuid.NewString(), fmt.Sprintf("idempotency round %d", round) + type attempt struct { + status int + id string + instance string + } + attempts := make([]attempt, parallel) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range parallel { + wg.Go(func() { + own := newClient(baseURL) + <-start + status, msg, instance, err := own.sendWithKey(room.ID, token, key, text) + if err == nil { + attempts[i] = attempt{status, msg.ID, instance} + } + }) + } + close(start) + wg.Wait() + same := true + for _, a := range attempts { + if a.instance != "" { + instances[a.instance] = true + } + if a.status != http.StatusCreated || a.id == "" || a.id != attempts[0].id { + same = false + } + } + if same { + r.SameMessageRounds++ + } + } + r.InstancesSeen = len(instances) + status, _, _, err := newClient(baseURL).sendWithKey(room.ID, token, key, text+" changed") + if err != nil { + return r, err + } + r.Conflict409 = status == http.StatusConflict + listed, _, err := readHistory(api, room.ID, token) + if err != nil { + return r, err + } + r.History = len(listed) + r.Passed = r.SameMessageRounds == rounds && + r.History == rounds && + r.Conflict409 && + (!requireReplicas || r.InstancesSeen >= 2) + return r, nil +} + +// clientPassword is shared by the synthetic clients of one run. +const clientPassword = "concurrency-check-password" + +// registerClients registers and logs in n clients, returning their ids and +// bearer tokens in the same order. +func registerClients(api *client, runID string, n int) (ids, tokens []string, err error) { + for i := range n { + var created user.Response + username := fmt.Sprintf("load_%s_%d", runID, i) + body := map[string]string{ + "username": username, + "display_name": fmt.Sprintf("Load client %d", i), + "password": clientPassword, + } + if err := api.post("/api/v1/users", "", body, &created); err != nil { + return nil, nil, fmt.Errorf("register client %d: %w", i, err) + } + token, err := api.login(username, clientPassword) + if err != nil { + return nil, nil, fmt.Errorf("log in client %d: %w", i, err) + } + ids = append(ids, created.ID) + tokens = append(tokens, token) + } + return ids, tokens, nil +} + +func createChat(api *client, runID string, members []string, creatorToken string) (chat.ChatResponse, error) { + var created chat.ChatResponse + body := map[string]any{"title": "Concurrency check " + runID, "member_ids": members} + if err := api.post("/api/v1/chats", creatorToken, body, &created); err != nil { + return chat.ChatResponse{}, fmt.Errorf("create chat: %w", err) + } + return created, nil +} + +// sendConcurrently gives every member its own HTTP client and releases them +// together, so the server sees genuinely parallel writes to one chat. +func sendConcurrently(baseURL, chatID string, tokens []string, perClient int) (map[string]struct{}, int) { + var mu sync.Mutex + created := make(map[string]struct{}, len(tokens)*perClient) + failed := 0 + + var wg sync.WaitGroup + start := make(chan struct{}) + for i, token := range tokens { + wg.Add(1) + go func(n int, token string) { + defer wg.Done() + own := newClient(baseURL) + <-start + for m := range perClient { + var msg chat.MessageResponse + err := own.postMessage(chatID, token, fmt.Sprintf("client %d message %d", n, m), &msg) + mu.Lock() + if err != nil { + failed++ + } else { + created[msg.ID] = struct{}{} + } + mu.Unlock() + } + }(i, token) + } + close(start) + wg.Wait() + return created, failed +} + +func readHistory(api *client, chatID, token string) ([]string, int, error) { + var ids []string + pages := 0 + cursor := "" + for { + var page pagination.PageResponse[chat.MessageResponse] + if err := api.get("/api/v1/chats/"+chatID+"/messages", token, cursor, &page); err != nil { + return nil, pages, fmt.Errorf("read history: %w", err) + } + pages++ + for _, m := range page.Items { + ids = append(ids, m.ID) + } + if page.NextCursor == nil { + return ids, pages, nil + } + cursor = *page.NextCursor + } +} + +// compare returns how many distinct ids the history holds and how many +// acknowledged ids never showed up in it. +func compare(created map[string]struct{}, listed []string) (unique, missing int) { + seen := make(map[string]struct{}, len(listed)) + for _, id := range listed { + seen[id] = struct{}{} + } + for id := range created { + if _, ok := seen[id]; !ok { + missing++ + } + } + return len(seen), missing +} diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go new file mode 100644 index 00000000..f0e1b045 --- /dev/null +++ b/tools/concurrency/main_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "os" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func TestRunAgainstLiveServer(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 10, 10, 5, 4, false) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Clients != 10 || report.PerClient != 10 || report.Expected != 100 { + t.Fatalf("report = %+v", report) + } + if report.Created != 100 || report.Listed != 100 || report.Unique != 100 || report.Missing != 0 { + t.Fatalf("report = %+v", report) + } + if !report.Ordered || !report.Passed { + t.Fatalf("report = %+v", report) + } + idem := report.Idempotency + if idem.Rounds != 5 || idem.Parallel != 4 || idem.SameMessageRounds != 5 || idem.History != 5 || !idem.Conflict409 || !idem.Passed { + t.Fatalf("idempotency = %+v", idem) + } + if idem.InstancesSeen != 1 { + t.Fatalf("one in-process server must be seen as one instance: %+v", idem) + } +} + +// Through the balancer the same key must reach at least two replicas; one +// replica alone does not prove the idempotency is shared. +func TestIdempotencyRequiresTwoReplicasByDefault(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 2, 1, 2, 2, true) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Idempotency.Passed || report.Passed { + t.Fatalf("a single instance passed the shared idempotency check: %+v", report.Idempotency) + } +} + +func TestRunFailsOnUnreachableServer(t *testing.T) { + if _, err := run("http://127.0.0.1:1", 2, 2, 1, 1, false); err == nil { + t.Fatal("expected an error for an unreachable server") + } +} diff --git a/tools/loadgen/client.go b/tools/loadgen/client.go new file mode 100644 index 00000000..fce2c380 --- /dev/null +++ b/tools/loadgen/client.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// result is one HTTP exchange; instance is the X-Instance-Id of the answer, +// empty when the balancer answered itself or nothing answered. +type result struct { + status int + instance string + body []byte + latency time.Duration + err error +} + +type client struct { + base string + http *http.Client + timeout time.Duration +} + +// newClient keeps many connections open to the balancer, as many concurrent +// users would; HAProxy balances every request, not every connection. +func newClient(base string, timeout time.Duration) *client { + transport := &http.Transport{ + MaxIdleConns: 10000, + MaxIdleConnsPerHost: 10000, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + } + return &client{base: base, http: &http.Client{Transport: transport}, timeout: timeout} +} + +func (c *client) do(ctx context.Context, method, path, token, key string, body []byte, basic string) result { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return result{err: err} + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + switch { + case basic != "": + req.Header.Set("Authorization", "Basic "+basic) + case token != "": + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + start := time.Now() + resp, err := c.http.Do(req) + if err != nil { + return result{err: err, latency: time.Since(start)} + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + r := result{status: resp.StatusCode, instance: resp.Header.Get("X-Instance-Id"), body: payload, latency: time.Since(start), err: err} + return r +} + +func (c *client) postJSON(ctx context.Context, path, token, key string, body any) result { + raw, err := json.Marshal(body) + if err != nil { + return result{err: err} + } + return c.do(ctx, http.MethodPost, path, token, key, raw, "") +} + +// expect decodes the body of a result with the wanted status. +func expect(r result, want int, out any) error { + if r.err != nil { + return r.err + } + if r.status != want { + return fmt.Errorf("status %d, want %d: %s", r.status, want, r.body) + } + if out == nil { + return nil + } + return json.Unmarshal(r.body, out) +} + +func (c *client) register(ctx context.Context, username, password string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]string{"username": username, "display_name": username, "password": password} + err := expect(c.postJSON(ctx, "/api/v1/users", "", "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) login(ctx context.Context, username, password string) (string, error) { + var session struct { + Token string `json:"token"` + } + basic := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + err := expect(c.do(ctx, http.MethodPost, "/api/v1/auth/sessions", "", "", nil, basic), http.StatusCreated, &session) + return session.Token, err +} + +func (c *client) createChat(ctx context.Context, token, title string, members []string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]any{"title": title, "member_ids": members} + err := expect(c.postJSON(ctx, "/api/v1/chats", token, "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) send(ctx context.Context, chatID, token, key, text string) result { + return c.postJSON(ctx, "/api/v1/chats/"+chatID+"/messages", token, key, map[string]string{"text": text}) +} + +// firstPage reads the oldest page of a history: the read of the profile. +func (c *client) firstPage(ctx context.Context, chatID, token string) result { + return c.do(ctx, http.MethodGet, "/api/v1/chats/"+chatID+"/messages?limit=50", token, "", nil, "") +} + +type message struct { + ID string `json:"id"` + Text string `json:"text"` +} + +// history reads a whole chat, page by page. +func (c *client) history(ctx context.Context, chatID, token string) ([]message, error) { + var all []message + path := "/api/v1/chats/" + chatID + "/messages?limit=100" + for range 100000 { + var page struct { + Items []message `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if err := expect(c.do(ctx, http.MethodGet, path, token, "", nil, ""), http.StatusOK, &page); err != nil { + return nil, err + } + all = append(all, page.Items...) + if page.NextCursor == nil { + return all, nil + } + path = "/api/v1/chats/" + chatID + "/messages?limit=100&cursor=" + url.QueryEscape(*page.NextCursor) + } + return nil, fmt.Errorf("history of %s did not end", chatID) +} diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go new file mode 100644 index 00000000..2873bfee --- /dev/null +++ b/tools/loadgen/main.go @@ -0,0 +1,343 @@ +// Command loadgen is the open-loop load generator of the experiments. +// +// loadgen bench — seeds data, warms up, then offers each rate of -rates +// for -step as a mix of history reads and message sends. +// loadgen timeline — one user sends messages at -rate, retrying uncertain +// sends with the same Idempotency-Key, then reads the +// history back: every confirmed message once. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + mrand "math/rand/v2" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" +) + +type config struct { + Mode string `json:"mode"` + BaseURL string `json:"base_url"` + Users int `json:"users"` + Chats int `json:"chats"` + SeedMessages int `json:"seed_messages_per_chat"` + ReadRatio float64 `json:"read_ratio"` + Rates []float64 `json:"rates"` + Step string `json:"step"` + Warmup string `json:"warmup"` + Rate float64 `json:"rate"` + Duration string `json:"duration"` + Timeout string `json:"client_timeout"` + Seed uint64 `json:"seed"` + Scrape bool `json:"scrape"` + StopOnFail bool `json:"stop_on_fail"` + StatsURL string `json:"stats_url"` + APIHost string `json:"api_host"` + DebugPort string `json:"debug_port"` + Out string `json:"-"` + + step, warmup, duration, timeout time.Duration +} + +func main() { + if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") { + fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags") + os.Exit(2) + } + cfg, err := parse(os.Args[1], os.Args[2:]) + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if cfg.Mode == "bench" { + err = runBench(ctx, cfg) + } else { + err = runTimeline(ctx, cfg) + } + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(1) + } +} + +func parse(mode string, args []string) (config, error) { + cfg := config{Mode: mode} + fs := flag.NewFlagSet(mode, flag.ContinueOnError) + rates := fs.String("rates", "200,400", "bench: offered requests per second of each step") + fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API") + fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users") + fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members") + fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring") + fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends") + fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step") + fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate") + fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second") + fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") + fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") + fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99") + fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") + fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") + fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") + fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)") + fs.StringVar(&cfg.Out, "out", "/out", "directory for the results") + if err := fs.Parse(args); err != nil { + return cfg, err + } + for _, part := range strings.Split(*rates, ",") { + r, err := strconv.ParseFloat(strings.TrimSpace(part), 64) + if err != nil || r <= 0 { + return cfg, fmt.Errorf("rates: %q is not a positive number", part) + } + cfg.Rates = append(cfg.Rates, r) + } + for _, d := range []struct { + name string + raw string + dst *time.Duration + }{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} { + v, err := time.ParseDuration(d.raw) + if err != nil || v < 0 { + return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw) + } + *d.dst = v + } + if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 { + return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive") + } + return cfg, os.MkdirAll(cfg.Out, 0o755) +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0o644) +} + +// startScrape returns the function that stops the sampling and waits. +func startScrape(ctx context.Context, cfg config, start time.Time) func() { + if !cfg.Scrape { + return func() {} + } + ctx, cancel := context.WithCancel(ctx) + s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}} + var wg sync.WaitGroup + wg.Go(func() { + if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil { + fmt.Fprintln(os.Stderr, "loadgen: scrape:", err) + } + }) + return func() { cancel(); wg.Wait() } +} + +type benchReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Users int `json:"users"` + Chats int `json:"chats"` + Seeded int `json:"seeded_messages"` + Steps []stepSummary `json:"steps"` + WritesOK int `json:"messages_written_during_steps"` +} + +func runBench(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages) + if err != nil { + return err + } + report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages} + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + + // next runs on the scheduling goroutine, which owns the random source. + counter := 0 + next := func(rec *recorder) func(time.Time) func() { + return func(planned time.Time) func() { + r := ds.rooms[rng.IntN(len(ds.rooms))] + a := ds.actors[r.members[rng.IntN(len(r.members))]] + write := rng.Float64() >= cfg.ReadRatio + counter++ + text := fmt.Sprintf("load %d", counter) + return func() { + sent := time.Now() + var res result + if write { + res = c.send(ctx, r.id, a.token, uuid.NewString(), text) + } else { + res = c.firstPage(ctx, r.id, a.token) + } + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write}) + } + } + } + if cfg.warmup > 0 { + openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{})) + } + all := &recorder{} + for _, rate := range cfg.Rates { + if ctx.Err() != nil { + break + } + stepStart := time.Now() + rec := &recorder{} + openLoop(ctx, rate, cfg.step, next(rec)) + s := summarize(rate, cfg.step, rec.all()) + s.StartedAt = stepStart.Sub(start).Seconds() + report.Steps = append(report.Steps, s) + report.WritesOK += s.Writes + for _, x := range rec.all() { + all.add(x) + } + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) + if cfg.StopOnFail && s.overloaded() { + break + } + } + stopScrape() + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { + return err + } + return writeJSON(filepath.Join(cfg.Out, "steps.json"), report) +} + +type timelineReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Jobs int `json:"messages_attempted"` + Attempts int `json:"requests"` + Confirmed int `json:"confirmed"` + RetriedJobs int `json:"messages_retried"` + Unresolved int `json:"unresolved"` + Rejected int `json:"rejected_4xx"` + Classes map[string]int `json:"classes"` + Instances map[string]int `json:"instances"` + History int `json:"history_messages"` + Duplicates int `json:"duplicates"` + Missing int `json:"missing_confirmed"` + Passed bool `json:"passed"` +} + +// retryDelay spaces the attempts of one message whose outcome was uncertain. +const retryDelay = 200 * time.Millisecond + +// retryGrace bounds the retries of an uncertain message after the schedule ends. +const retryGrace = 10 * time.Second + +func runTimeline(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) + ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0) + if err != nil { + return err + } + r := ds.rooms[0] + sender := ds.actors[r.members[0]] + report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}} + runID := randomHex(3) + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + deadline := start.Add(cfg.duration + retryGrace) + + var mu sync.Mutex + confirmed := map[string]string{} // text -> message id + rec := &recorder{} + jobs := 0 + openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() { + jobs++ + key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs) + return func() { + for attempt := 0; ; attempt++ { + sent := time.Now() + res := c.send(ctx, r.id, sender.token, key, text) + cls := classify(res) + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true}) + mu.Lock() + switch { + case cls == "ok": + var m message + if json.Unmarshal(res.body, &m) == nil { + confirmed[text] = m.ID + } + case res.err == nil && res.status >= 400 && res.status < 500: + report.Rejected++ + case time.Now().After(deadline) || ctx.Err() != nil: + report.Unresolved++ + default: + if attempt == 0 { + report.RetriedJobs++ + } + mu.Unlock() + time.Sleep(retryDelay) + planned = time.Now() + continue + } + mu.Unlock() + return + } + } + }) + stopScrape() + samples := rec.all() + report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed) + for _, x := range samples { + report.Classes[x.class]++ + if x.instance != "" { + report.Instances[x.instance]++ + } + } + + history, err := c.history(ctx, r.id, sender.token) + if err != nil { + return fmt.Errorf("read history back: %w", err) + } + report.History = len(history) + seen := map[string]int{} + byID := map[string]string{} + for _, m := range history { + seen[m.Text]++ + byID[m.ID] = m.Text + } + for _, n := range seen { + if n > 1 { + report.Duplicates += n - 1 + } + } + for text, id := range confirmed { + if byID[id] != text { + report.Missing++ + } + } + report.Passed = report.Duplicates == 0 && report.Missing == 0 + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil { + return err + } + if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil { + return err + } + fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n", + report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing) + if !report.Passed { + return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing) + } + return nil +} diff --git a/tools/loadgen/main_test.go b/tools/loadgen/main_test.go new file mode 100644 index 00000000..4f6c8c75 --- /dev/null +++ b/tools/loadgen/main_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func liveServer(t *testing.T) string { + return serve(t, nil) +} + +func serve(t *testing.T, wrap func(http.Handler) http.Handler) string { + t.Helper() + a := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "loadgen-test", + }) + var h http.Handler = a.Engine + if wrap != nil { + h = wrap(h) + } + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return srv.URL +} + +// Registration hashes a password with Argon2id, which is slow on a small +// replica; the setup is not measured, so it must not fail on the short +// timeout of the measured requests. +func TestSlowSetupDoesNotFailTheRun(t *testing.T) { + slowAccounts := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && (r.URL.Path == "/api/v1/users" || r.URL.Path == "/api/v1/auth/sessions") { + time.Sleep(300 * time.Millisecond) + } + next.ServeHTTP(w, r) + }) + } + cfg, err := parse("bench", []string{"-base-url", serve(t, slowAccounts), "-users", "2", "-chats", "1", "-seed-messages", "1", + "-rates", "10", "-step", "1s", "-warmup", "0s", "-timeout", "100ms", "-out", t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatalf("a setup slower than -timeout failed the run: %v", err) + } +} + +func TestBenchAgainstLiveServer(t *testing.T) { + out := t.TempDir() + cfg, err := parse("bench", []string{"-base-url", liveServer(t), "-users", "4", "-chats", "2", "-seed-messages", "3", + "-rates", "20,40", "-step", "1s", "-warmup", "0s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report benchReport + raw, err := os.ReadFile(filepath.Join(out, "steps.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("steps.json: %v %s", err, raw) + } + if len(report.Steps) != 2 || report.Seeded != 6 || report.Users != 4 || report.Chats != 2 { + t.Fatalf("report = %+v", report) + } + for _, s := range report.Steps { + if s.Sent != int(s.OfferedRPS) || s.OK != s.Sent || s.ErrorShare != 0 || s.Instances["loadgen-test"] != s.Sent { + t.Fatalf("step = %+v", s) + } + } + csv, err := os.ReadFile(filepath.Join(out, "seconds.csv")) + if err != nil || !strings.HasPrefix(string(csv), "second,sent,ok,") || strings.Count(string(csv), "\n") < 3 { + t.Fatalf("seconds.csv: %v\n%s", err, csv) + } +} + +func TestTimelineVerifiesHistory(t *testing.T) { + out := t.TempDir() + cfg, err := parse("timeline", []string{"-base-url", liveServer(t), "-rate", "20", "-duration", "1s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runTimeline(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report timelineReport + raw, err := os.ReadFile(filepath.Join(out, "timeline.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("timeline.json: %v %s", err, raw) + } + if report.Jobs != 20 || report.Confirmed != 20 || report.History != 20 || report.Duplicates != 0 || report.Missing != 0 || !report.Passed { + t.Fatalf("report = %+v", report) + } +} diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go new file mode 100644 index 00000000..7a84f6a2 --- /dev/null +++ b/tools/loadgen/run.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "sync" + "time" +) + +// openLoop sends request i at start + i/rate whatever the response times: a +// closed loop would let a slow system lower its own load and hide its latency +// (coordinated omission). It returns once every job has finished. +func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { + n := int(rate * d.Seconds()) + interval := time.Duration(float64(time.Second) / rate) + start := time.Now() + var wg sync.WaitGroup + for i := range n { + planned := start.Add(time.Duration(i) * interval) + if wait := time.Until(planned); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + if ctx.Err() != nil { + break + } + wg.Go(next(planned)) + } + wg.Wait() +} + +// recorder collects samples from concurrent jobs. +type recorder struct { + mu sync.Mutex + samples []sample +} + +func (r *recorder) add(s sample) { + r.mu.Lock() + r.samples = append(r.samples, s) + r.mu.Unlock() +} + +func (r *recorder) all() []sample { + r.mu.Lock() + defer r.mu.Unlock() + return append([]sample(nil), r.samples...) +} diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go new file mode 100644 index 00000000..6b95db75 --- /dev/null +++ b/tools/loadgen/scrape.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "io" + "net" + "net/http" + "os" + "strings" + "time" +) + +// scraper samples the HAProxy statistics and every replica's expvar once a +// second; replicas are found through Docker DNS, as HAProxy finds them. +type scraper struct { + statsURL string + apiHost string + debugPort string + http *http.Client +} + +type scrapeLine struct { + T float64 `json:"t"` + HAProxy []map[string]string `json:"haproxy,omitempty"` + Replicas []json.RawMessage `json:"replicas,omitempty"` +} + +// haproxyFields are the columns of the statistics CSV that are kept. +var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"} + +func (s scraper) run(ctx context.Context, start time.Time, path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)} + if err := enc.Encode(line); err != nil { + return err + } + } +} + +func (s scraper) get(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := s.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) +} + +// haproxy keeps the rows of the replicas backend. +func (s scraper) haproxy(ctx context.Context) []map[string]string { + raw, err := s.get(ctx, s.statsURL) + if err != nil { + return nil + } + rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll() + if err != nil || len(rows) < 2 { + return nil + } + header := rows[0] + col := map[string]int{} + for i, name := range header { + col[name] = i + } + var out []map[string]string + for _, row := range rows[1:] { + if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") { + continue + } + kept := map[string]string{} + for _, name := range haproxyFields { + if i, ok := col[name]; ok { + kept[name] = row[i] + } + } + out = append(out, kept) + } + return out +} + +// replicas returns the expvar documents of the running replicas, trimmed to +// the fields the study uses. +func (s scraper) replicas(ctx context.Context) []json.RawMessage { + addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost) + if err != nil { + return nil + } + var out []json.RawMessage + for _, addr := range addrs { + raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars") + if err != nil { + continue + } + var vars struct { + Instance string `json:"instance"` + GOMAXPROCS int `json:"gomaxprocs"` + Pool json.RawMessage `json:"pgxpool"` + Mem struct { + HeapAlloc uint64 `json:"HeapAlloc"` + Sys uint64 `json:"Sys"` + NumGC uint32 `json:"NumGC"` + PauseTotalNs uint64 `json:"PauseTotalNs"` + GCCPUFraction float64 `json:"GCCPUFraction"` + } `json:"memstats"` + } + if json.Unmarshal(raw, &vars) != nil { + continue + } + kept, err := json.Marshal(map[string]any{ + "addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS, + "pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys, + "num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction, + }) + if err == nil { + out = append(out, kept) + } + } + return out +} diff --git a/tools/loadgen/seed.go b/tools/loadgen/seed.go new file mode 100644 index 00000000..61624236 --- /dev/null +++ b/tools/loadgen/seed.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + mrand "math/rand/v2" + "sync" + "time" + + "github.com/google/uuid" +) + +// seedPassword is shared by the synthetic users of one run. +const seedPassword = "loadgen-synthetic-password" + +// seedTimeout bounds one setup request. The setup is not measured, and +// registration hashes a password with Argon2id, slow on a small replica. +const seedTimeout = 30 * time.Second + +// seedWorkers bounds the parallel registrations: each one hashes a password +// with Argon2id, and the setup must not be what the study measures. +const seedWorkers = 8 + +type actor struct { + id, token string +} + +type room struct { + id string + members []int // indexes into dataset.actors +} + +type dataset struct { + actors []actor + rooms []room + // messages is the number of messages written while seeding. + messages int +} + +// seed registers users and logs each in once, creates chats of two to five +// random members and writes perChat messages into each. Logins happen here, +// so the password hashing never shows up in the measured steps. +func seed(ctx context.Context, c *client, rng *mrand.Rand, users, chats, perChat int) (dataset, error) { + if users < 2 || chats < 1 { + return dataset{}, fmt.Errorf("need at least 2 users and 1 chat") + } + runID := randomHex(3) + ds := dataset{actors: make([]actor, users)} + if err := parallel(users, func(i int) error { + name := fmt.Sprintf("lg_%s_%d", runID, i) + id, err := c.register(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("register %s: %w", name, err) + } + token, err := c.login(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("log in %s: %w", name, err) + } + ds.actors[i] = actor{id: id, token: token} + return nil + }); err != nil { + return dataset{}, err + } + + for j := range chats { + size := 2 + rng.IntN(4) + members := rng.Perm(users)[:min(size, users)] + ids := make([]string, len(members)) + for k, m := range members { + ids[k] = ds.actors[m].id + } + id, err := c.createChat(ctx, ds.actors[members[0]].token, fmt.Sprintf("load %s %d", runID, j), ids) + if err != nil { + return dataset{}, fmt.Errorf("create chat %d: %w", j, err) + } + ds.rooms = append(ds.rooms, room{id: id, members: members}) + } + + if err := parallel(len(ds.rooms), func(j int) error { + r := ds.rooms[j] + for k := range perChat { + sender := ds.actors[r.members[k%len(r.members)]] + res := c.send(ctx, r.id, sender.token, uuid.NewString(), fmt.Sprintf("seed %d", k)) + if err := expect(res, 201, nil); err != nil { + return fmt.Errorf("seed message into chat %d: %w", j, err) + } + } + return nil + }); err != nil { + return dataset{}, err + } + ds.messages = len(ds.rooms) * perChat + return ds, nil +} + +// parallel runs fn(0..n-1) on seedWorkers goroutines and returns the first +// error. +func parallel(n int, fn func(i int) error) error { + var ( + wg sync.WaitGroup + mu sync.Mutex + first error + ) + next := make(chan int) + for range seedWorkers { + wg.Go(func() { + for i := range next { + if err := fn(i); err != nil { + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + } + } + }) + } + for i := range n { + next <- i + } + close(next) + wg.Wait() + return first +} + +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go new file mode 100644 index 00000000..8f5dd8ca --- /dev/null +++ b/tools/loadgen/stats.go @@ -0,0 +1,205 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "slices" + "sort" + "strings" + "time" +) + +// maxValidLag: if requests left later than this at p99, the generator, not +// the system, limited the step. +const maxValidLag = 10 * time.Millisecond + +// The capacity criterion of the study: a step passes when it is valid, has +// fewer than 1 % errors and its successful requests have a p99 within 500 ms. +const ( + maxErrorShare = 0.01 + sloP99 = 500 * time.Millisecond +) + +// sample is one request as the generator saw it. +type sample struct { + planned time.Time + lag time.Duration // actual send time minus planned send time + latency time.Duration + class string + instance string + write bool +} + +// classify names the outcome of a request. Errors of the balancer itself carry +// no X-Instance-Id and get the suffix _lb; client_timeout is the generator +// giving up, not a timeout of the system. +func classify(r result) string { + switch { + case r.err != nil && isTimeout(r.err): + return "client_timeout" + case r.err != nil: + return "conn_error" + case r.status >= 200 && r.status < 300: + return "ok" + case r.instance == "": + return fmt.Sprintf("http_%d_lb", r.status) + default: + return fmt.Sprintf("http_%d", r.status) + } +} + +func isTimeout(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// percentile is the nearest-rank percentile of an ascending slice. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + rank := int(math.Ceil(p / 100 * float64(len(sorted)))) + return sorted[min(max(rank, 1), len(sorted))-1] +} + +type stepSummary struct { + OfferedRPS float64 `json:"offered_rps"` + Seconds float64 `json:"seconds"` + StartedAt float64 `json:"started_at_s"` + Sent int `json:"sent"` + OK int `json:"ok"` + OKRPS float64 `json:"ok_rps"` + ErrorShare float64 `json:"error_share"` + Classes map[string]int `json:"classes"` + P50Ms float64 `json:"ok_p50_ms"` + P95Ms float64 `json:"ok_p95_ms"` + P99Ms float64 `json:"ok_p99_ms"` + LagP99Ms float64 `json:"lag_p99_ms"` + Valid bool `json:"valid"` + Passed bool `json:"passed"` + Instances map[string]int `json:"instances"` + Writes int `json:"writes_ok"` +} + +// summarize reduces one step; latency percentiles cover successful requests. +func summarize(rate float64, d time.Duration, samples []sample) stepSummary { + s := stepSummary{ + OfferedRPS: rate, + Seconds: d.Seconds(), + Sent: len(samples), + Classes: map[string]int{}, + Instances: map[string]int{}, + } + var ok, lags []time.Duration + for _, x := range samples { + s.Classes[x.class]++ + lags = append(lags, x.lag) + if x.instance != "" { + s.Instances[x.instance]++ + } + if x.class == "ok" { + ok = append(ok, x.latency) + if x.write { + s.Writes++ + } + } + } + slices.Sort(ok) + slices.Sort(lags) + s.OK = len(ok) + s.OKRPS = float64(s.OK) / d.Seconds() + if s.Sent > 0 { + s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent) + } + s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)) + lagP99 := percentile(lags, 99) + s.LagP99Ms = ms(lagP99) + s.Valid = lagP99 <= maxValidLag + s.Passed = s.Valid && !s.overloaded() + return s +} + +func (s stepSummary) overloaded() bool { + return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99) +} + +func ms(d time.Duration) float64 { + return float64(d.Microseconds()) / 1000 +} + +// errorColumns are the fixed error columns of the per-second CSV; any other +// class lands in err_other. +var errorColumns = []string{"client_timeout", "conn_error"} + +// secondsCSV renders the per-second timeline, attributed by planned send +// time so it lines up with the moment a fault was injected. +func secondsCSV(start time.Time, samples []sample) string { + type bucket struct { + samples []sample + } + buckets := map[int]*bucket{} + last := 0 + for _, x := range samples { + sec := int(x.planned.Sub(start) / time.Second) + if buckets[sec] == nil { + buckets[sec] = &bucket{} + } + buckets[sec].samples = append(buckets[sec].samples, x) + last = max(last, sec) + } + var b strings.Builder + b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n") + for sec := 0; sec <= last; sec++ { + bk := buckets[sec] + if bk == nil { + fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec) + continue + } + var ok, lags []time.Duration + counts := map[string]int{} + instances := map[string]int{} + for _, x := range bk.samples { + lags = append(lags, x.lag) + if x.instance != "" { + instances[x.instance]++ + } + switch { + case x.class == "ok": + ok = append(ok, x.latency) + case slices.Contains(errorColumns, x.class): + counts[x.class]++ + case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"): + counts["http_5xx_lb"]++ + case strings.HasPrefix(x.class, "http_5"): + counts["http_5xx_api"]++ + case strings.HasPrefix(x.class, "http_4"): + counts["http_4xx"]++ + default: + counts["err_other"]++ + } + } + slices.Sort(ok) + slices.Sort(lags) + names := make([]string, 0, len(instances)) + for name := range instances { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, name := range names { + parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name])) + } + fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n", + sec, len(bk.samples), len(ok), + counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"], + ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)), + strings.Join(parts, "|")) + } + return b.String() +} diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go new file mode 100644 index 00000000..f9a8bf1b --- /dev/null +++ b/tools/loadgen/stats_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" +) + +type timeoutErr struct{} + +func (timeoutErr) Error() string { return "i/o timeout" } +func (timeoutErr) Timeout() bool { return true } +func (timeoutErr) Temporary() bool { return true } + +var _ net.Error = timeoutErr{} + +func TestClassify(t *testing.T) { + for _, c := range []struct { + res result + want string + }{ + {result{status: 201, instance: "a"}, "ok"}, + {result{status: 200, instance: "a"}, "ok"}, + {result{status: 409, instance: "a"}, "http_409"}, + {result{status: 503, instance: "a"}, "http_503"}, + {result{status: 503}, "http_503_lb"}, + {result{status: 502}, "http_502_lb"}, + {result{err: context.DeadlineExceeded}, "client_timeout"}, + {result{err: fmt.Errorf("post: %w", timeoutErr{})}, "client_timeout"}, + {result{err: errors.New("connection refused")}, "conn_error"}, + } { + if got := classify(c.res); got != c.want { + t.Errorf("classify(%+v) = %q, want %q", c.res, got, c.want) + } + } +} + +func TestPercentileNearestRank(t *testing.T) { + ms := func(v ...int) []time.Duration { + out := make([]time.Duration, len(v)) + for i, x := range v { + out[i] = time.Duration(x) * time.Millisecond + } + return out + } + sorted := ms(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + for p, want := range map[float64]time.Duration{50: 5 * time.Millisecond, 95: 10 * time.Millisecond, 99: 10 * time.Millisecond, 10: time.Millisecond} { + if got := percentile(sorted, p); got != want { + t.Errorf("p%v = %v, want %v", p, got, want) + } + } + if got := percentile(nil, 99); got != 0 { + t.Errorf("empty p99 = %v, want 0", got) + } +} + +// A step is invalid when the generator itself fell behind its schedule: the +// system was then offered less than the nominal rate. +func TestStepSummaryValidityAndErrorShare(t *testing.T) { + samples := []sample{ + {class: "ok", latency: 10 * time.Millisecond, lag: time.Millisecond}, + {class: "ok", latency: 20 * time.Millisecond, lag: time.Millisecond}, + {class: "client_timeout", latency: 2 * time.Second, lag: time.Millisecond}, + {class: "http_503_lb", latency: time.Millisecond, lag: time.Millisecond}, + } + s := summarize(4, time.Second, samples) + if s.Sent != 4 || s.OK != 2 || s.OKRPS != 2 || s.ErrorShare != 0.5 || !s.Valid { + t.Fatalf("summary = %+v", s) + } + if s.Classes["client_timeout"] != 1 || s.Classes["http_503_lb"] != 1 { + t.Fatalf("classes = %v", s.Classes) + } + if s.P99Ms != 20 { + t.Fatalf("p99 of successful requests = %v ms, want 20", s.P99Ms) + } + samples[0].lag = 50 * time.Millisecond + samples[1].lag = 50 * time.Millisecond + if s := summarize(4, time.Second, samples); s.Valid { + t.Fatalf("lagging generator marked valid: %+v", s) + } +} + +// Open loop: requests leave on schedule even when responses are slow. A +// closed loop with one worker would send only a handful here. +func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { + var fired atomic.Int32 + start := time.Now() + openLoop(context.Background(), 50, time.Second, func(time.Time) func() { + return func() { + fired.Add(1) + time.Sleep(200 * time.Millisecond) + } + }) + if n := fired.Load(); n < 45 || n > 50 { + t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) + } + // About 1.2 s; the bound leaves room for -race on a shared CI runner and + // still fails a closed loop, which would need 50 x 200 ms = 10 s. + if took := time.Since(start); took > 2500*time.Millisecond { + t.Fatalf("open loop took %v, want about 1.2 s", took) + } +} + +func TestStepPassesOnlyWithinTheObjective(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, fast); !s.Passed { + t.Fatalf("fast clean step must pass: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); s.Passed { + t.Fatalf("p99 above 500 ms must fail: %+v", s) + } + failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond}) + if s := summarize(3, time.Second, failing); s.Passed { + t.Fatalf("errors above 1 %% must fail: %+v", s) + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.Passed { + t.Fatalf("an invalid step must fail: %+v", s) + } +} + +// The ladder stops when the system leaves the objective, not when the +// generator alone fell behind: a later step may still be valid and pass. +func TestOnlyTheSystemEndsTheLadder(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.overloaded() { + t.Fatalf("a lagging generator must not end the ladder: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); !s.overloaded() { + t.Fatalf("p99 above 500 ms must end the ladder: %+v", s) + } + failing := []sample{ok(10*time.Millisecond, time.Millisecond), {class: "client_timeout", lag: time.Millisecond}} + if s := summarize(2, time.Second, failing); !s.overloaded() { + t.Fatalf("errors above 1 %% must end the ladder: %+v", s) + } +} diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go new file mode 100644 index 00000000..83fe48cb --- /dev/null +++ b/tools/reconcilebench/main.go @@ -0,0 +1,395 @@ +// Command reconcilebench measures reconciliation against disposable real +// PostgreSQL and S3 services, independently of the compose stand. +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func main() { + o, err := parseOptions(os.Args[1:], os.Stderr) + if errors.Is(err, flag.ErrHelp) { + return + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + var records bytes.Buffer + var resultOut io.Writer = os.Stdout + if o.output != "" { + resultOut = io.MultiWriter(os.Stdout, &records) + } + err = exercise(ctx, o, resultOut, os.Stderr) + if o.output != "" { + if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil { + err = errors.Join(err, fmt.Errorf("save results: %w", saveErr)) + } + } + cancel() + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "reconcilebench:", err) + os.Exit(1) + } +} + +func writeResults(path string, records []byte) error { + decoder := json.NewDecoder(bytes.NewReader(records)) + rows := make([]json.RawMessage, 0) + for { + var row json.RawMessage + err := decoder.Decode(&row) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + rows = append(rows, row) + } + raw, err := json.MarshalIndent(rows, "", " ") + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0644) +} + +type measurement struct { + Phase string `json:"phase"` + Mode string `json:"mode"` + TargetFiles int `json:"target_files"` + GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"` + Stats attachment.Stats `json:"stats"` + Findings []attachment.Finding `json:"findings,omitempty"` +} + +// Heap samples cover this Go process during Run. They exclude service processes +// and are deliberately not described as process RSS or total deployment memory. +func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) { + done := make(chan struct{}) + peakCh := make(chan uint64, 1) + go func() { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + peak := mem.HeapAlloc + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + case <-done: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + peakCh <- peak + return + } + } + }() + m := measurement{Phase: phase, Mode: mode, TargetFiles: n} + options := attachment.ReconcileOptions{Apply: apply, Deep: deep} + stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) + close(done) + m.GoHeapPeakBytes = <-peakCh + m.Stats = stats + if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil { + return stats, writeErr + } + return stats, err +} + +func exercise(ctx context.Context, o options, out, progress io.Writer) error { + db, err := testenv.StartPostgres(ctx) + if err != nil { + return err + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + return err + } + defer s3.Close() + if err = json.NewEncoder(out).Encode(map[string]any{ + "phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image, + "grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(), + "memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3", + }); err != nil { + return err + } + f, err := seedFaults(ctx, db.Pool, s3.Store) + if err != nil { + return err + } + fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait) + if err = waitForAge(ctx, o.wait, progress); err != nil { + return err + } + if err = f.seedFresh(ctx); err != nil { + return err + } + r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace) + if err != nil { + return err + } + for _, run := range []struct { + mode string + apply bool + repairs, findings int64 + }{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} { + stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out) + if err != nil { + return err + } + if stats.Repaired != run.repairs || stats.Findings != run.findings { + return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats) + } + if err = f.assertState(ctx, run.apply); err != nil { + return fmt.Errorf("fault %s: %w", run.mode, err) + } + } + fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved") + // These resources were created above solely for this run; no compose + // endpoint or pre-existing database/bucket is ever accepted by this tool. + if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files CASCADE`); err != nil { + return err + } + if err = emptyBucket(ctx, s3.Store); err != nil { + return err + } + seeded := 0 + for _, n := range o.sizes { + for i := seeded; i < n; i++ { + if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil { + return err + } + if (i+1)%500 == 0 { + fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1) + } + } + seeded = n + stats, err := measure(ctx, r, "cost", "dry", n, false, false, out) + if err != nil { + return err + } + if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 { + return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats) + } + } + return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes}) +} + +func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error { + start := time.Now() + timer := time.NewTimer(d) + defer timer.Stop() + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + case <-ticker.C: + fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d) + } + } +} + +type benchFile struct { + hash []byte + before, after string +} +type benchObject struct { + key string + after bool +} +type faultFixture struct { + db *pgxpool.Pool + blobs attachment.Blobs + user, chat uuid.UUID + files []benchFile + objects []benchObject +} + +func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) { + f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()} + if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil { + return nil, err + } + if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil { + return nil, err + } + if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil { + return nil, err + } + orphan := sha256.Sum256([]byte("old orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil { + return nil, err + } + for _, c := range []struct { + label, status, after string + used, present, corrupt bool + }{ + {"pending", "pending", "", false, true, false}, + {"unused", "ready", "", false, true, false}, + {"used", "ready", "ready", true, true, false}, + {"corrupt", "ready", "ready", true, true, true}, + {"missing", "ready", "failed", true, false, false}, + {"already_failed", "failed", "failed", true, false, false}, + } { + if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil { + return nil, err + } + } + return f, nil +} + +func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error { + if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + f.objects = append(f.objects, benchObject{key: key, after: keep}) + return nil +} + +func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error { + body := []byte("fixture/" + label) + hash := sha256.Sum256(body) + created := time.Now().Add(-2 * time.Hour) + if fresh { + created = time.Now() + } + if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil { + return err + } + f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after}) + if used { + id, message := uuid.New(), uuid.New() + if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil { + return err + } + } + if present { + if corrupt { + body[0] ^= 1 + } + return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "") + } + return nil +} + +func (f *faultFixture) seedFresh(ctx context.Context) error { + if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil { + return err + } + orphan := sha256.Sum256([]byte("fresh orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil { + return err + } + return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true) +} + +func (f *faultFixture) assertState(ctx context.Context, applied bool) error { + for _, file := range f.files { + var status string + err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + want := file.before + if applied { + want = file.after + } + if status != want { + return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want) + } + } + for _, obj := range f.objects { + _, err := f.blobs.Stat(ctx, obj.key) + want := !applied || obj.after + if err != nil && !errors.Is(err, attachment.ErrObjectMissing) { + return err + } + if (err == nil) != want { + return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want) + } + } + var attachments, links int + if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil { + return err + } + if attachments != 4 || links != 4 { + return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links) + } + return nil +} + +func emptyBucket(ctx context.Context, blobs attachment.Blobs) error { + for { + page, err := blobs.List(ctx, "", "", 500) + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + for _, obj := range page { + if err := blobs.Delete(ctx, obj.Key); err != nil { + return err + } + } + } +} + +func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error { + body := fmt.Sprintf("scale/%09d", i) + hash := sha256.Sum256([]byte(body)) + if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + _, err := db.Exec(ctx, `WITH f AS ( + INSERT INTO files(hash,size_bytes,mime_type,status,created_at) + VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user) + return err +} diff --git a/tools/reconcilebench/main_test.go b/tools/reconcilebench/main_test.go new file mode 100644 index 00000000..e77ecf01 --- /dev/null +++ b/tools/reconcilebench/main_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestOptionsProtectRealAgingAndValidateSizes(t *testing.T) { + got, err := parseOptions(nil, io.Discard) + if err != nil || got.grace != 2*time.Minute || got.wait != 125*time.Second || !reflect.DeepEqual(got.sizes, []int{100, 1000, 5000}) { + t.Fatalf("defaults: %+v %v", got, err) + } + for _, args := range [][]string{{"--grace=1m"}, {"--wait=2m"}, {"--sizes=0"}, {"--sizes=10,10"}, {"--sizes=10,5"}, {"--sizes=oops"}, {"extra"}} { + if _, err := parseOptions(args, io.Discard); err == nil { + t.Errorf("accepted unsafe options %v", args) + } + } + got, err = parseOptions([]string{"--sizes=2,7", "--grace=3m", "--wait=3m2s"}, io.Discard) + if err != nil || !reflect.DeepEqual(got.sizes, []int{2, 7}) { + t.Fatalf("explicit options: %+v %v", got, err) + } +} + +func TestOutputOptionWritesStructuredJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "track2", "results.json") + o, err := parseOptions([]string{"--output", path}, io.Discard) + if err != nil || o.output != path { + t.Fatalf("output flag: %+v %v", o, err) + } + if err = writeResults(path, []byte("{\"phase\":\"fault\"}\n{\"phase\":\"complete\"}\n")); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err = json.Unmarshal(raw, &rows); err != nil || len(rows) != 2 || rows[1]["phase"] != "complete" { + t.Fatalf("results JSON: %s %v", raw, err) + } +} diff --git a/tools/reconcilebench/options.go b/tools/reconcilebench/options.go new file mode 100644 index 00000000..5e6344e8 --- /dev/null +++ b/tools/reconcilebench/options.go @@ -0,0 +1,43 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +type options struct { + grace, wait time.Duration + sizes []int + output string +} + +func parseOptions(args []string, out io.Writer) (options, error) { + var o options + f := flag.NewFlagSet("reconcilebench", flag.ContinueOnError) + f.SetOutput(out) + f.DurationVar(&o.grace, "grace", 2*time.Minute, "reconciler grace (minimum 2m)") + f.DurationVar(&o.wait, "wait", 125*time.Second, "actual S3 aging wait (must exceed grace by at least 1s)") + f.StringVar(&o.output, "output", "", "also save measurement records as a JSON array at this path") + sizes := f.String("sizes", "100,1000,5000", "strictly increasing referenced object counts") + if err := f.Parse(args); err != nil { + return o, err + } + if f.NArg() != 0 || o.grace < 2*time.Minute || o.wait-o.grace < time.Second { + return o, errors.New("no positional arguments; grace >= 2m and wait >= grace+1s required") + } + last := 0 + for _, raw := range strings.Split(*sizes, ",") { + n, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || n <= last { + return o, fmt.Errorf("sizes must be strictly increasing positive integers: %q", raw) + } + o.sizes = append(o.sizes, n) + last = n + } + return o, nil +}