diff --git a/.env.example b/.env.example index d99d26a..34af899 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,74 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env for your LOCAL synthetic test environment; +# Replace every "change-me" value before starting the stack. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +DB_POOL_MAX_CONNS=10 + +# Local PostgreSQL. The API receives a full DATABASE_URL built by compose.yaml. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-postgres +POSTGRES_DB=messenger + +# Redis is the only accepted shared session store. AOF keeps absolute TTLs. +SESSION_STORE=redis + +# HAProxy is the only public API entrypoint. API replicas use expose, not ports. +LB_PORT=8080 +HAPROXY_STATS_PORT=8404 + +# RustFS root credentials are used ONLY by the one-shot s3-init bootstrap. +# The API and reconciler receive the scoped S3_ACCESS_KEY_ID / +# S3_SECRET_ACCESS_KEY pair below, whose policy is limited to attachments/*. +S3_ROOT_ACCESS_KEY=change-me-rustfs-root +S3_ROOT_SECRET_KEY=change-me-rustfs-root-secret +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-app-secret +S3_APP_USER=messenger-api +S3_APP_POLICY=messenger-attachments-rw +S3_ENDPOINT=http://rustfs:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key + +# Attachments data limits MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Weighted decode budget shared by concurrent validates +IMAGE_DECODE_MEMORY_MB=256 + +# Stale pending rows and orphan objects older than the grace period are removed. +RECONCILE_GRACE_SECONDS=900 +RECONCILE_INTERVAL_SECONDS=300 + +# Per-replica and other services' resource limits. +# API_REPLICAS must stay within the 16 backend slots declared by +# server-template in infra/haproxy/haproxy.cfg. +API_REPLICAS=2 +API_CPU_LIMIT=0.50 +API_MEM_LIMIT=512M +POSTGRES_CPU_LIMIT=1.0 +POSTGRES_MEM_LIMIT=512M +REDIS_CPU_LIMIT=0.50 +REDIS_MEM_LIMIT=256M +RUSTFS_CPU_LIMIT=1.0 +RUSTFS_MEM_LIMIT=512M +RECONCILER_CPU_LIMIT=0.25 +RECONCILER_MEM_LIMIT=128M +HAPROXY_CPU_LIMIT=0.50 +HAPROXY_MEM_LIMIT=128M + +# K6 for detailed tests. +K6_VUS=10 +K6_DURATION=60s +K6_CPU_LIMIT=1.0 +K6_MEM_LIMIT=512M + +# Optional overrides instead of the compose-generated URLs. A *_FILE value is +# read from a file, which is how Docker/Compose secrets are usually mounted. +# DATABASE_URL=postgres://messenger:password@localhost:5432/messenger?sslmode=disable +# DATABASE_URL_FILE=/run/secrets/database_url +# CACHE_URL=redis://localhost:6379/0 +# CACHE_URL_FILE=/run/secrets/cache_url +# S3_ENDPOINT=http://localhost:9000 diff --git a/Dockerfile b/Dockerfile index a4b7b3c..ce530bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,17 @@ WORKDIR /src COPY app/go.mod app/go.sum ./ RUN go mod download COPY app/ ./ -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconciler ./cmd/reconciler FROM alpine:3.22 -RUN addgroup -S -g 10001 app && adduser -S -u 10001 -G app app +RUN apk add --no-cache ca-certificates \ + && addgroup -S -g 10001 app \ + && adduser -S -u 10001 -G app app COPY --from=build /out/server /usr/local/bin/server +COPY --from=build /out/reconciler /usr/local/bin/reconciler USER app EXPOSE 8080 diff --git a/Makefile b/Makefile index 5d86d17..0d6322a 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,27 @@ PYTHON ?= python3 BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= +COMPOSE ?= docker compose --env-file .env +VUS ?= 10 +DURATION ?= 60s TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover \ + test-concurrency test-attachments test-upload-memory test-reconcile \ + test-private-bucket test-attachment-failover test-interrupted-upload \ + app-test app-build up down wait k6 help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API" + @echo "test-concurrency: cross-replica idempotency race" + @echo "test-attachments: graded negative upload/access cases" + @echo "test-persistence/test-failover: explicit operator hooks" + @echo "test-upload-memory: >=10 concurrent uploads with sampled API memory" + @echo "test-reconcile: stale pending, orphan object and ready-attachment safety" + @echo "test-private-bucket: anonymous denial and scoped S3 credential" + @echo "test-attachment-failover: upload, stop serving replica, download elsewhere" + @echo "test-interrupted-upload: half-sent upload leaves no row/object/temp file" + @echo "up/down/wait: compose lifecycle; app-test/app-build: Go checks" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +43,44 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-concurrency: + $(PYTHON) tests/custom/idempotency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) tests/custom/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-upload-memory: + $(PYTHON) scripts/measure-upload-memory.py + +test-reconcile: + ./scripts/verify-reconcile.sh + +test-private-bucket: + ./scripts/check-private-bucket.sh + +test-attachment-failover: + ./scripts/verify-attachment-failover.sh + +test-interrupted-upload: + ./scripts/verify-interrupted-upload.sh + +app-test: + cd app && go test ./... + +app-build: + cd app && go build ./... + +up: + $(COMPOSE) up -d --build + +down: + $(COMPOSE) down + +wait: + @echo "waiting for $(BASE_URL)/health/ready" + @until curl -fsS "$(BASE_URL)/health/ready" >/dev/null; do sleep 1; done + @echo "ready" + +k6: + $(COMPOSE) --profile load run --rm k6 run -e VUS=$(VUS) -e DURATION=$(DURATION) --summary-export /results/last.json /scripts/messages.js diff --git a/REPORT.md b/REPORT.md index 9ad2b84..c6e33bd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,374 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: заполняется студентом перед сдачей (ФИО, группа). +- Целевая оценка: **4** (уровень 3 и все пункты уровня 4). Дополнительный + исследовательский трек не заявлен. +- Commit/PR: локальная ветка поверх `68968c4`; ссылка на PR добавляется после + публикации. +- Использованные источники, библиотеки и помощь ИИ: контракт курса + (`contracts/`), публичные тесты (`tests/`), документация RustFS, AWS SDK for + Go v2, Fiber v3, pgx. ИИ-ассистент использовался для генерации и ревью кода, + текста отчёта и тестовых сценариев; все результаты ниже получены на реально + запущенном стенде. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +### Требования + +- Linux с Docker Engine 29.9.1 и Docker Compose v5.6.0 (проверено на этих + версиях; нужен обычный bridge network и named volumes). +- Не менее ~5 ГБ свободного места: образы Go builder, RustFS, PostgreSQL, + Redis, HAProxy, Python и volumes. При переполнении диска стенд не считается + исправным. +- `make`, `curl`, Python 3.12 для HTTP-проверок. Go для локальных проверок: + модуль объявляет `go 1.27.1`, Dockerfile использует + `golang:1.27.1-alpine`; локальный `go1.25.6` собирает проект через + авто-загрузку toolchain. +- Версии образов закреплены: `rustfs/rustfs:1.0.4`, `rustfs/rc:v0.1.36`, + `postgres:16.10-alpine`, `redis:7.4.7-alpine`, `haproxy:3.2.6-alpine`, + `python:3.12-alpine`, `alpine:3.22`, `grafana/k6:1.3.0`. + +### Подготовка и запуск + +```sh +git clone && cd mtusi2026lab4 + +cp .env.example .env +# Отредактировать .env: POSTGRES_PASSWORD, S3_ROOT_ACCESS_KEY, +# S3_ROOT_SECRET_KEY, S3_SECRET_ACCESS_KEY. Значения не коммитить. +# S3_ACCESS_KEY_ID по умолчанию messenger-api; это имя scoped IAM-user, +# а не root-пользователь RustFS. + +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +``` + +Порядок запуска: `postgres`, `redis`, `rustfs` → healthchecks → одноразовый +`s3-init` (bucket, private access, IAM-user и policy) → `migrate` (встроенные +миграции) → `api` (2 реплики) и `reconciler` → `haproxy`. API запускаются с +`RUN_MIGRATIONS=false`, поэтому миграции не гоняются репликами. + +Если позже изменить `S3_SECRET_ACCESS_KEY` в `.env`, bootstrap нужно повторить +вручную: `docker compose run --rm --no-deps -T s3-init`. IAM-user обновится, +API продолжит работать с новым секретом. + +Готовность: + +```sh +curl -fsS http://localhost:8080/health/ready +``` + +Если shell не видит опубликованные порты Docker (например, агент запущен в +отдельном контейнере), проверки выполняются из контейнера в сети стенда: + +```sh +docker run --rm --network mtusi2026lab_default -v "$PWD:/work:ro" -w /work \ + python:3.12-alpine python3 tests/smoke.py --base-url http://haproxy:8080 +``` + +Для hook-тестов (`persistence`, `failover`) удобнее подключить сам shell к сети +стенда и использовать `BASE_URL=http://haproxy:8080`: + +```sh +docker network connect mtusi2026lab_default +``` + +Это изменение только тестового окружения; в приложении `localhost:8080` +остаётся штатным адресом при опубликованном порте. + +### Проверки + +```sh +make check +python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python + +cd app && go vet ./... && go test ./... && go build ./... && cd .. + +make test BASE_URL=http://localhost:8080 +make test-persistence BASE_URL=http://localhost:8080 ACTION_SCRIPT=scripts/restart.sh +make test-failover BASE_URL=http://localhost:8080 ACTION_SCRIPT=scripts/stop-one.sh +make test-concurrency BASE_URL=http://localhost:8080 +make test-attachments BASE_URL=http://localhost:8080 +make test-upload-memory +make test-reconcile +make test-private-bucket +make test-attachment-failover +make test-interrupted-upload +``` + +Если `localhost` недоступен, заменить `BASE_URL` на `http://haproxy:8080` и +выполнять из контейнера/сети стенда, как описано выше. `make test-failover` +намеренно оставляет одну реплику остановленной; восстановление: + +```sh +docker compose --env-file .env up -d --scale api=2 api +``` + +### Остановка + +```sh +docker compose --env-file .env down # volumes сохраняются +docker compose --env-file .env down -v # намеренное удаление учебных данных +``` + +`down -v` удаляет PostgreSQL, Redis и RustFS volumes. При смене +`POSTGRES_PASSWORD` на уже существующем `pgdata` PostgreSQL не пустит API: +нужно либо вернуть прежний пароль, либо осознанно пересоздать volume на +отдельном стенде. Это проверено на практике. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент + | + v +HAProxy :8080 (единственный опубликованный вход, stats :8404) + | + +--> api-1 (Go/Fiber, expose 8080) --+ + +--> api-2 (Go/Fiber, expose 8080) --+--> PostgreSQL 16.10 (метаданные, истина) + +--> Redis 7.4 (сессии, AOF) + +--> RustFS 1.0.4 (приватные байты) + | + reconciler (pending/orphan cleanup) + +s3-init (one-shot): bucket + anonymous=private + IAM messenger-api + policy +migrate (one-shot): встроенные миграции 0001..0003 +``` + +| Роль | Где | Ответственность | +| --- | --- | --- | +| Вход | `haproxy` | round-robin, health `GET /health/ready`, 16 слотов DNS | +| API | `api` × 2 | stateless HTTP, авторизация, streaming upload/download | +| Метаданные | `postgres` | пользователи, чаты, сообщения, idempotency, attachments | +| Сессии | `redis` | SHA-256 digest токена, absolute TTL | +| Байты | `rustfs` | private bucket, object key назначает сервер | +| Bootstrap | `s3-init` | идемпотентные bucket/IAM/policy | +| Сверка | `reconciler` | stale pending и orphan objects после grace period | + +### Что перенесено из lab3 + +Сохранены пользователи, Basic-login, сессии в Redis, чаты, сообщения, +идемпотентность POST сообщений, пагинация, HAProxy, две API-реплики, +readiness с PostgreSQL/Redis, graceful shutdown, `X-Request-Id` и +`X-Instance-Id`, а также форматы ошибок. Текстовые сообщения остаются +валидными; `attachment_ids` по умолчанию `[]`. + +### Что добавлено в lab4 + +- Метаданные вложения хранятся в PostgreSQL, байты — только в RustFS. В API нет + ни локального постоянного файла оригинала, ни публичного URL bucket. +- Object key: `attachments//`. Имя файла от клиента + используется только для отображения и никогда не становится путём. +- Загрузка: авторизация по членству в чате → проверка filename и + Content-Type → ограниченное чтение тела во временный файл с SHA-256 → + MIME/DecodeConfig/full decode → `pending` row → S3 PUT → `ready` row → 201. + 201 означает, что сохранены и объект, и метаданные. +- Если процесс упал после `pending`, но до `ready`, reconciler удаляет и row, и + объект после grace period. Если объект остался без row (крайне узкое окно + или ручное вмешательство), reconciler удаляет orphan. +- `/health/ready` требует PostgreSQL, Redis и S3 `HeadBucket`; при SIGTERM + сначала выставляется флаг shutdown и readiness начинает отдавать 503. +- Bucket не опубликован, console выключена. API и reconciler используют + отдельного IAM-user с policy только на `attachments/*`; root-credential есть + только у `s3-init`. +- `/health/ready` и все ответы API по-прежнему содержат `X-Request-Id` и + `X-Instance-Id`. + +### Как посмотреть bucket без публичного порта + +S3 API и admin console не опубликованы, поэтому осмотр и проверки выполняются +одноразовым контейнером в сети стенда. Тот же приём используют +`scripts/check-private-bucket.sh` и `scripts/verify-reconcile.sh`: + +```sh +docker run --rm --network mtusi2026lab_default --env-file .env -e HOME=/tmp \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc ls "app/$S3_BUCKET/attachments/"' +``` + +Контейнер использует scoped-ключ API, а не root; для административных действий +(например, повторного bootstrap) есть отдельный root-ключ у `s3-init`. + +### Транзакционные границы и согласованность + +1. Валидация и чтение тела заканчиваются до создания строки. Обрыв соединения + не создаёт ни row, ни object. +2. `INSERT ... status='pending'` коммитится до S3 PUT. Это намеренно: при + падении процесса остаётся указатель, по которому reconciler найдёт объект. +3. S3 PUT выполняется с `Content-Length` и `Content-Type`; при ошибке — + компенсирующее удаление объекта и pending-строки. +4. `UPDATE ... SET status='ready'` выполняется после успешного PUT. Ошибка + обновления также запускает компенсацию. +5. Сообщение и `message_attachments` вставляются в одной транзакции с + idempotency-записью. Все вложения проверяются на принадлежность чату и + статус `ready` в той же транзакции. +6. Fingerprint идемпотентности: для сообщений без вложений сохранён старый + `sha256(text)`; с вложениями добавлены канонически отсортированные UUID. +7. Reconciler работает с grace period (по умолчанию 900 с), поэтому свежая + загрузка не может быть удалена как «зависшая». + +### Лимиты, таймауты и память + +| Механизм | Значение | Где | +| --- | --- | --- | +| Максимальный размер оригинала | 10 MiB = 10 485 760 байт | config + SQL constraint | +| Максимум пикселей | 40 000 000 | config, проверяется по header до decode | +| Глобальный HTTP body | 10 MiB + 1 MiB | Fiber `BodyLimit` | +| JSON body | 1 MiB | `common.BindJSON` | +| Копирование upload | 32 KiB buffer, `io.LimitReader` | handler | +| Бюджет decode | 256 MiB weighted semaphore | `attachments/image.go` | +| Upload/download timeout | 30 с / 60 с | handler context | +| S3 health | 2 с | `HeadBucket` | +| Pending/orphan grace | 900 с | reconciler | + +Тело не читается через `io.ReadAll` для upload и не буферизуется Fiber +(`StreamRequestBody: true`). Даже отклонённый по `Content-Length` запрос +дочитывается только в пределах `max + 1 MiB`, чтобы клиент успел получить 413, +а не connection reset. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Все команды выполнены на запущенном стенде 2026-10-31; smoke, негативные +сценарии, reconciler и interrupted upload повторены 2026-11-06 после +рефакторинга с тем же результатом. `BASE_URL` в лог-файлах — +`http://haproxy:8080` (shell подключён к сети стенда). -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke lab4 | `make test BASE_URL=http://haproxy:8080` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` | +| S3 в общем стенде, приватный bucket | `make test-private-bucket` | anonymous GET → 403; scoped PUT/HEAD в `attachments/` разрешён, вне prefix и CreateBucket запрещены | `evidence/private-bucket.txt` | +| Идемпотентный bootstrap | `docker compose run --rm --no-deps -T s3-init` дважды | оба запуска `S3 bootstrap complete`, exit 0 | `evidence/s3-bootstrap.txt` | +| Upload 201 ready + checksum | public smoke | `status=ready`, `variants=[]`, `error_code=null`, SHA-256 и size совпали, download байт-в-байт | `evidence/smoke.txt` | +| Доступ участника и запрет постороннему | `make test-attachments` | bob скачал оригинал; eve получила 404 на metadata и bytes | `evidence/attachments-negative.txt` | +| Слишком большой файл | `make test-attachments` | 10 MiB + 1 → 413, без 5xx | `evidence/attachments-negative.txt` | +| Повреждённое изображение | `make test-attachments` | PNG-сигнатура с битым телом → 415 | `evidence/attachments-negative.txt` | +| Подмена MIME | `make test-attachments` | PNG с `Content-Type: image/jpeg` → 415 | `evidence/attachments-negative.txt` | +| Чужой attachment_id | `make test-attachments` | 404 для metadata/content; чужой чат → 404 | `evidence/attachments-negative.txt` | +| Вложение другого чата | `make test-attachments` | привязка к другому чату → 404 в обе стороны | `evidence/attachments-negative.txt` | +| Пользовательская ошибка не 500 | `make test-attachments` | все негативные сценарии 400/404/413/415 | `evidence/attachments-negative.txt` | +| Upload → остановка реплики → download с другой | `make test-attachment-failover` | upload принят `ad8141b5e17c`, download обслужен `2229551f45ac`, SHA-256 совпал | `evidence/attachment-failover.txt` | +| Обрыв загрузки | `make test-interrupted-upload` | rows 52 → 52, objects 52 → 52, temp files 0 | `evidence/interrupted-upload.txt` | +| Reconciliation: stale pending | `make test-reconcile` | dry-run ничего не изменил; реальный прогон `deleted_rows=1 deleted_objects=1` | `evidence/reconcile.txt` | +| Reconciliation: fresh pending | `make test-reconcile` | строка и объект остались при `grace=1s` | `evidence/reconcile.txt` | +| Reconciliation: orphan object | `make test-reconcile` | после удаления row объект удалён `deleted_objects=1` | `evidence/reconcile.txt` | +| Reconciliation: ready не трогается | `make test-reconcile` | row и объект ready-вложения остались | `evidence/reconcile.txt` | +| ≥10 одновременных загрузок, пик RAM | `make test-upload-memory` | 10 × 7 563 763 B, 2 реплики, peak 354.1 MiB total, 174.1/180.1 MiB на реплику при лимите 512 MiB | `evidence/upload-memory.json`, `evidence/upload-memory-run.txt` | +| Persistence после restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` PostgreSQL/Redis/RustFS/HAProxy/API изменился, данные и сессия сохранились | `evidence/persistence.txt`, `evidence/persistence-containers.txt` | +| Failover одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 2 реплики до отказа, 118 успешных записей, 1 переходная ошибка, recovery 2.557 с, дублей нет | `evidence/failover.txt` | +| Конкурентная идемпотентность | `make test-concurrency` | 12 запросов, 2 реплики, одна запись | `evidence/idempotency.txt` | +| Readiness отражает PostgreSQL | stop/start `postgres`, запрос `/health/ready` внутри API | 503 при остановке, 200 после восстановления | `evidence/readiness-postgres.txt` | +| Readiness отражает Redis | stop/start `redis` | 503 при остановке, 200 после восстановления | `evidence/readiness-redis.txt` | +| Readiness отражает S3 | stop/start `rustfs` | 503 при остановке, 200 после восстановления | `evidence/readiness-s3.txt` | +| Graceful shutdown | SIGTERM/`docker stop` при restart/failover | процесс пишет `shutting down`, readiness 503, соединения завершаются; hooks укладываются в лимит | `evidence/persistence.txt`, `evidence/failover.txt` | +| Offline-контракт | `make check`, `make validate` | оба PASS | вывод команд в этом отчёте | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Профиль стенда -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Docker на Linux, Compose bridge network, лимиты из `.env`: +API — 0.5 CPU / 512 MiB на реплику, PostgreSQL — 1.0 CPU / 512 MiB, +Redis — 0.5 CPU / 256 MiB, RustFS — 1.0 CPU / 512 MiB, +HAProxy — 0.5 CPU / 128 MiB. Это учебный стенд, а не production-измерение; +результаты зависят от хоста и общего Docker daemon. + +### Нагрузка POST сообщений + +`python3 tests/load.py --base-url http://haproxy:8080 --duration 15 --concurrency 8 --output evidence/load.json` + +| Показатель | Значение | +| --- | --- | +| Длительность / concurrency | 15.216 с / 8 | +| Успешные / ошибки | 6249 / 0 | +| Successful RPS | 410.689 | +| p50 / p95 / p99 | 18.104 / 31.058 / 46.769 мс | + +Это closed-loop генератор: он страдает coordinated omission, не делает +прозрачных retry и измеряет только успешные HTTP-подтверждения. Числа +показывают, что стенд не деградировал после добавления S3, но не являются +пределом пропускной способности. + +### Память при одновременных загрузках + +Методика: 10 потоков одновременно отправляют один и тот же сгенерированный +PNG 1800×1400 (7 563 763 байт, 2.52 Мп) через HAProxy; host-side скрипт +опрашивает Docker Engine stats каждые 50 мс и суммирует `memory_stats.usage` +двух API-контейнеров. Перед прогоном API перезапущены, чтобы baseline не +содержал page cache прошлых тестов. + +| Показатель | Значение | +| --- | --- | +| Baseline total | 14.7 MiB | +| Peak total | 354.1 MiB | +| Peak на реплику | 174.1 / 180.1 MiB | +| Лимит на реплику | 512 MiB | +| Время 10 загрузок | 2.60 с | +| Проверка download | SHA-256 совпал | + +Одиночная загрузка того же файла на свежем стенде дала peak 280.4 MiB total +(142.9/137.5 MiB на контейнер). Рост при переходе от 1 к 10 одновременным +загрузкам существенно меньше 10×: тело не буферизуется целиком, decode +ограничен семафором. Docker stats включает page cache, поэтому абсолютные +значения выше, чем «чистая» heap-память Go. + +### Persistence + +`scripts/restart.sh` перезапускает `postgres redis rustfs haproxy api` без +удаления volumes. В `evidence/persistence-containers.txt` видно, что +`StartedAt` всех пяти сервисов изменился, а публичный тест после этого снова +прочитал исходное сообщение, сессию и оригинал вложения. Это подтверждает, что +hook действительно перезапустил процессы, а не только вернул 0. + +### Failover + +`scripts/stop-one.sh` сопоставляет `TARGET_INSTANCE_ID` с hostname контейнера и +оставляет ровно эту реплику остановленной. Публичный тест наблюдал 2 реплики, +отправил 118 успешных записей за 15 секунд, допустил 1 переходную ошибку, +зафиксировал восстановление за 2.557 с и не нашёл дублей. Отдельный +`verify-attachment-failover.sh` загрузил файл через одну реплику, остановил её +и скачал тот же SHA-256 через другую. + +### Матрица отказов + +| Воздействие | Ожидание | Наблюдение | Восстановление / потери | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановлена одна API-реплика | HAProxy исключает её, вторая продолжает | 2 реплики до, 118 успешных записей, 1 переходная ошибка | 2.557 с; подтверждённые данные не потеряны | +| Upload, затем остановка принявшей реплики | Файл доступен через другую | download обслужила другая реплика, SHA-256 совпал | без потерь, 1 попытка | +| Restart всего стенда | Сессии, сообщения и файлы сохраняются | persistence PASS; все `StartedAt` изменились | volumes сохранены, повторный логин не нужен | +| Остановлен RustFS | readiness 503, новые upload не принимаются | `/health/ready` → 503 | после старта → 200, данные в volume | +| Остановлен Redis | readiness 503 | `/health/ready` → 503 | после старта → 200; сессия пережила restart в persistence-тесте | +| Остановлен PostgreSQL | readiness 503 | `/health/ready` → 503 | после старта → 200; данные в volume | +| Обрыв HTTP-загрузки | Нет row/object/temp | rows и objects не изменились, temp files 0 | автоматически, без reconciler | +| Падение между pending и ready | reconciler удаляет пару после grace | dry-run + реальный прогон | до 900 с удержания, затем очистка | +| Orphan object без row | reconciler удаляет после grace | `deleted_objects=1` | до 900 с, затем очистка | +| Переполнение диска хоста | Не гарантируется | один прогон дал 500/503 (`No space left on device`) | после очистки build cache повторные прогоны зелёные; это ограничение стенда, не заявленная HA-функция | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Стенд не является production-HA: PostgreSQL, Redis и RustFS — одиночные + контейнеры с volume; HAProxy и API реплицированы, но хост и хранилища не + кластеризованы. Остановка БД или S3 — это отказ, а не прозрачное + переключение. +- Reconciler — safety net, а не мгновенная транзакция. Между S3 PUT и + `status=ready` существует окно, в котором объект уже есть, но 201 ещё не + возвращён; после grace периода такая пара удаляется. Клиент, не получивший + 201, должен повторить загрузку, что создаст новый object key. +- Grace period 900 с и интервал 300 с — учебные значения. Для реальной системы + их нужно согласовать с максимальной длительностью загрузки и SLA очистки. +- Лимит 10 MiB / 40 Мп и таймауты 30/60 с проверены на локальном стенде. На + медленном канале загрузка 10 MiB может не уложиться в 30 с; это осознанное + ограничение, а не обещание поддерживать любой RTT. +- Нет presigned URL, фоновой обработки, thumbnails и вариантов изображений: + контракт lab4 этого не требует, а обязательный proxy download оставлен + единственным способом доступа к байтам. +- Нет отдельного API для удаления вложений и сообщений; очистка выполняется + только reconciler-ом для abandoned данных. Пользовательские данные в + учебном стенде не удаляются автоматически. +- Нагрузка измерена closed-loop генератором; для честного throughput нужен + open-loop/k6 с фиксированным arrival rate и отдельным профилем upload. +- Проверка private bucket и scoped policy выполняется изнутри Compose network; + снаружи S3 API и console недоступны, что и требуется. +- Все секреты хранятся только в локальном `.env`; в отчёт, evidence и git они + не попадают. `CHANGELOG.md` теперь отслеживается, `.env` — нет. diff --git a/app/cmd/reconciler/main.go b/app/cmd/reconciler/main.go new file mode 100644 index 0000000..c2746ec --- /dev/null +++ b/app/cmd/reconciler/main.go @@ -0,0 +1,105 @@ +// Command reconciler periodically removes abandoned attachment objects and +// pending metadata rows. It is safe to run concurrently with the API and safe +// to re-run; see internal/reconcile for the grace-period rules. +package main + +import ( + "context" + "flag" + "log" + "log/slog" + "os/signal" + "syscall" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/database" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/reconcile" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" +) + +func main() { + once := flag.Bool("once", false, "run one reconciliation pass and exit") + dryRun := flag.Bool("dry-run", false, "log what would be removed without changing either store") + grace := flag.Duration("grace", 0, "override RECONCILE_GRACE_SECONDS for this run") + interval := flag.Duration("interval", 0, "override RECONCILE_INTERVAL_SECONDS for this run") + flag.Parse() + + cfg, err := config.Load() + if err != nil { + log.Fatalf("configuration: %v", err) + } + if *grace > 0 { + cfg.ReconcileGrace = *grace + } + if *interval > 0 { + cfg.ReconcileInterval = *interval + } + + rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + pool, err := database.Connect(rootCtx, cfg.DatabaseURL, cfg.DBMaxConns, cfg.StartupWait) + if err != nil { + log.Fatalf("database: %v", err) + } + defer pool.Close() + + objects, err := objectstore.New(objectstore.Config{ + Endpoint: cfg.S3Endpoint, + Region: cfg.S3Region, + Bucket: cfg.S3Bucket, + AccessKey: cfg.S3AccessKey, + SecretKey: cfg.S3SecretKey, + }) + if err != nil { + log.Fatalf("object storage: %v", err) + } + + worker := reconcile.New( + store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL), + objects, + cfg.ReconcileGrace, + slog.Default(), + ) + + if *once { + stats, runErr := worker.RunOnce(rootCtx, *dryRun) + logStats(stats, *dryRun) + if runErr != nil { + log.Fatalf("reconcile: %v", runErr) + } + return + } + + for { + stats, runErr := worker.RunOnce(rootCtx, *dryRun) + if runErr != nil { + slog.Error("reconcile pass failed", "err", runErr) + } else { + logStats(stats, *dryRun) + } + + // A timer avoids the ticker behaviour where missed ticks accumulate: + // the next pass starts one full interval after this one finished. + timer := time.NewTimer(cfg.ReconcileInterval) + select { + case <-rootCtx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +func logStats(stats reconcile.Stats, dryRun bool) { + slog.Info("reconcile pass complete", + "dry_run", dryRun, + "stale_pending", stats.StalePending, + "orphan_objects", stats.OrphanObjects, + "deleted_rows", stats.DeletedRows, + "deleted_objects", stats.DeletedObjects, + "skipped", stats.Skipped, + ) +} diff --git a/app/cmd/server/main.go b/app/cmd/server/main.go index 9ec872a..e6f0be2 100755 --- a/app/cmd/server/main.go +++ b/app/cmd/server/main.go @@ -11,6 +11,7 @@ import ( "time" "gitflic.ru/project/bulla-max/mtusi2026lab/internal" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/attachments" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/database" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/health" @@ -49,7 +50,12 @@ func main() { } defer sessions.Close() - app := internal.New(cfg, pool, sessions) + attachments.CleanupStaleUploads(cfg.UploadTempDir, time.Hour) + + app, err := internal.New(cfg, pool, sessions) + if err != nil { + log.Fatalf("application: %v", err) + } addr := net.JoinHostPort(cfg.Host, cfg.Port) listenErr := make(chan error, 1) diff --git a/app/go.mod b/app/go.mod index c58def5..8657c1a 100755 --- a/app/go.mod +++ b/app/go.mod @@ -3,15 +3,34 @@ module gitflic.ru/project/bulla-max/mtusi2026lab go 1.27.1 require ( + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.13 + github.com/aws/aws-sdk-go-v2/credentials v1.20.13 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 + github.com/aws/smithy-go v1.28.7 github.com/gofiber/fiber/v3 v3.5.0 github.com/golang-migrate/migrate/v4 v4.20.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 github.com/redis/go-redis/v9 v9.23.0 golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.24.0 ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/gofiber/schema v1.8.8 // indirect github.com/gofiber/utils/v2 v2.6.1 // indirect @@ -29,7 +48,6 @@ require ( github.com/valyala/fasthttp v1.74.0 // indirect go.uber.org/atomic v1.12.0 // indirect golang.org/x/net v0.59.0 // indirect - golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.50.0 // indirect diff --git a/app/go.sum b/app/go.sum index 623cc7e..8f93f82 100755 --- a/app/go.sum +++ b/app/go.sum @@ -2,6 +2,42 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw= +github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -119,8 +155,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= diff --git a/app/internal/app.go b/app/internal/app.go index 4d3a0c2..c9f0c7e 100755 --- a/app/internal/app.go +++ b/app/internal/app.go @@ -5,14 +5,15 @@ package internal import ( "errors" "log/slog" - "time" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/attachments" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/auth" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/chats" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/health" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/messages" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/session" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/users" @@ -22,15 +23,33 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber.App { +func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber.App, error) { st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL) + objects, err := objectstore.New(objectstore.Config{ + Endpoint: cfg.S3Endpoint, + Region: cfg.S3Region, + Bucket: cfg.S3Bucket, + AccessKey: cfg.S3AccessKey, + SecretKey: cfg.S3SecretKey, + }) + if err != nil { + return nil, err + } + app := fiber.New(fiber.Config{ - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - IdleTimeout: 60 * time.Second, - BodyLimit: 1 * 1024 * 1024, - ErrorHandler: handleError, + ReadTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + IdleTimeout: cfg.IdleTimeout, + // BodyLimit is the coarse HTTP envelope. The attachment handler and + // BindJSON enforce their own smaller budgets. + BodyLimit: cfg.HTTPBodyLimit, + // Without StreamRequestBody fasthttp buffers the entire body + // before the handler runs. Streaming lets the upload path copy in + // 32 KiB chunks and stops a 10 MiB body from becoming a per-request + // allocation even when the request is later rejected. + StreamRequestBody: true, + ErrorHandler: handleError, }) app.Use(recover.New()) @@ -40,15 +59,25 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber. return c.Next() }) - healthHandler := health.NewHandler(st, sessions) + healthHandler := health.NewHandler(st, sessions, objects) healthRoutes := app.Group("/health") healthRoutes.Get("/live", healthHandler.Live) healthRoutes.Get("/ready", healthHandler.Ready) authHandler := auth.NewHandler(st, sessions) - usersHandler := users.NewHandler(st) - chatsHandler := chats.NewHandler(st) - messagesHandler := messages.NewHandler(st) + usersHandler := users.NewHandler(st, config.MaxJSONBodyBytes) + chatsHandler := chats.NewHandler(st, config.MaxJSONBodyBytes) + messagesHandler := messages.NewHandler(st, config.MaxJSONBodyBytes) + attachmentsHandler := attachments.NewHandler( + st, + objects, + attachments.NewValidator(cfg.MaxImagePixels, cfg.ImageDecodeBytes), + cfg.MaxUploadBytes, + cfg.UploadDrainLimit, + cfg.UploadTimeout, + cfg.DownloadTimeout, + cfg.UploadTempDir, + ) api := app.Group("/api/v1") api.Post("/users", usersHandler.CreateUser) @@ -62,8 +91,11 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber. protected.Get("/chats/:chat_id", chatsHandler.GetChat) protected.Get("/chats/:chat_id/messages", messagesHandler.ListMessages) protected.Post("/chats/:chat_id/messages", messagesHandler.CreateMessage) + protected.Post("/chats/:chat_id/attachments", attachmentsHandler.Upload) + protected.Get("/attachments/:attachment_id", attachmentsHandler.Get) + protected.Get("/attachments/:attachment_id/content", attachmentsHandler.Download) - return app + return app, nil } // handleError is the only writer of error responses: handlers and the store diff --git a/app/internal/attachments/handler.go b/app/internal/attachments/handler.go new file mode 100644 index 0000000..6fb8749 --- /dev/null +++ b/app/internal/attachments/handler.go @@ -0,0 +1,329 @@ +package attachments + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "os" + "strconv" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" + "github.com/gofiber/fiber/v3" + "github.com/google/uuid" +) + +const copyBufferSize = 32 * 1024 + +type Handler struct { + store *store.Store + objects *objectstore.Client + validator *Validator + maxUploadBytes int64 + drainLimit int64 + uploadTimeout time.Duration + downloadTimeout time.Duration + tempDir string +} + +func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler { + return &Handler{ + store: st, + objects: client, + validator: validator, + maxUploadBytes: maxUploadBytes, + drainLimit: drainLimit, + uploadTimeout: uploadTimeout, + downloadTimeout: downloadTimeout, + tempDir: tempDir, + } +} + +// Upload stores raw PNG/JPEG bytes. The order is deliberate: +// 1. authorize before reading the body; +// 2. stream the body to a bounded temporary file while hashing; +// 3. validate pixels and decode; +// 4. commit a pending metadata row; +// 5. PUT the object; +// 6. mark the row ready. +// +// A 201 is therefore returned only after both the object and its metadata are +// durable. Every failure between 4 and 6 has a compensation path, and the +// reconciler covers crashes that prevent compensation from running. +func (h *Handler) Upload(c fiber.Ctx) error { + userID := common.MustUserID(c) + + chatID, err := common.PathUUID(c, "chat_id") + if err != nil { + return err + } + + // Authorization precedes any body I/O: an outsider cannot make the server + // read a large upload before learning that the chat is not theirs. + isMember, err := h.store.IsMember(c.Context(), userID, chatID) + if err != nil { + return err + } + if !isMember { + return common.NotFound("chat not found") + } + + filename := c.Query("filename") + if err := common.ValidateFilename(filename); err != nil { + return err + } + + contentType, _, err := mime.ParseMediaType(c.Get(fiber.HeaderContentType)) + if err != nil || (contentType != "image/png" && contentType != "image/jpeg") { + return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg") + } + + // Reject an obviously oversized request before allocating a temp file or + // decoding anything. Chunked requests are caught by the streaming counter + // below. + // + // AGENT: the declared body is drained first, but only within the bounded + // HTTP envelope (upload limit plus the JSON body slack). If the server + // closed immediately, the client would often see a connection reset instead + // of the required 413 while it was still sending. Draining is bounded and + // uses a 32 KiB buffer, so it does not weaken the memory guarantee. + if rawLength := c.Get(fiber.HeaderContentLength); rawLength != "" { + declared, parseErr := strconv.ParseInt(rawLength, 10, 64) + if parseErr != nil || declared < 0 { + return common.BadRequest("invalid Content-Length") + } + if declared > h.maxUploadBytes { + if declared <= h.drainLimit { + _, _ = io.CopyBuffer(io.Discard, io.LimitReader(requestBody(c), declared), make([]byte, copyBufferSize)) + } + return common.PayloadTooLarge("upload exceeds the byte limit") + } + } + + if err = os.MkdirAll(h.tempDir, 0o700); err != nil { + slog.Error("create upload temp dir", "err", err) + return common.Internal() + } + file, err := os.CreateTemp(h.tempDir, "upload-*.part") + if err != nil { + slog.Error("create upload temp file", "err", err) + return common.Internal() + } + filePath := file.Name() + defer func() { + _ = file.Close() + _ = os.Remove(filePath) + }() + + // AGENT: the 32 KiB copy buffer is the only request-sized allocation here. + // The body is never read with io.ReadAll, so ten simultaneous 10 MiB + // uploads do not require ten 10 MiB Go slices. + var reader io.Reader = requestBody(c) + hasher := sha256.New() + copyBuffer := make([]byte, copyBufferSize) + written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, h.maxUploadBytes+1), copyBuffer) + if err != nil { + return common.BadRequest("incomplete upload body") + } + if written == 0 { + return common.UnsupportedMedia("upload body is empty") + } + if written > h.maxUploadBytes { + return common.PayloadTooLarge("upload exceeds the byte limit") + } + + ctx, cancel := context.WithTimeout(c.Context(), h.uploadTimeout) + defer cancel() + + info, err := h.validator.Validate(ctx, file, contentType) + if err != nil { + return err + } + + attachmentID := uuid.New() + attachment := model.Attachment{ + ID: attachmentID, + ChatID: chatID, + OwnerID: userID, + Filename: filename, + ContentType: info.ContentType, + SizeBytes: written, + SHA256: hex.EncodeToString(hasher.Sum(nil)), + Status: "pending", + Variants: []model.Variant{}, + ErrorCode: nil, + CreatedAt: common.NowUTC(), + UpdatedAt: common.NowUTC(), + // AGENT: the key is server-generated from UUIDs and never contains the + // client filename. That keeps traversal and collisions out of the + // object store even before application authorization is considered. + ObjectKey: objectKey(chatID, attachmentID), + } + + if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil { + return err + } + + if _, err = file.Seek(0, 0); err != nil { + h.compensate(attachment.ID, attachment.ObjectKey) + return common.BadRequest("cannot read upload body") + } + if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil { + slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID) + h.compensate(attachment.ID, attachment.ObjectKey) + return common.Unavailable("object storage unavailable") + } + + updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID) + if err != nil { + slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID) + h.compensate(attachment.ID, attachment.ObjectKey) + return common.Unavailable("attachment metadata unavailable") + } + + attachment.Status = "ready" + attachment.UpdatedAt = common.RFCFormTime(updatedAt) + return c.Status(fiber.StatusCreated).JSON(attachment) +} + +// compensate is best effort by design. If either delete fails, the pending row +// or the object remains visible to the reconciler, which retries after the +// configured grace period. +func (h *Handler) compensate(attachmentID uuid.UUID, key string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID) + } + if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil { + slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID) + } +} + +func (h *Handler) Get(c fiber.Ctx) error { + userID := common.MustUserID(c) + + attachmentID, err := common.PathUUID(c, "attachment_id") + if err != nil { + return err + } + + attachment, err := h.store.GetAttachmentForUser(c.Context(), userID, attachmentID) + if err != nil { + return err + } + if attachment.Status != "ready" { + return common.Conflict("attachment is not ready") + } + + return c.JSON(attachment) +} + +func (h *Handler) Download(c fiber.Ctx) error { + userID := common.MustUserID(c) + + attachmentID, err := common.PathUUID(c, "attachment_id") + if err != nil { + return err + } + + variant := c.Query("variant", "original") + if variant == "" { + variant = "original" + } + if variant != "original" { + return common.BadRequest("unsupported variant") + } + + attachment, err := h.store.GetAttachmentForUser(c.Context(), userID, attachmentID) + if err != nil { + return err + } + if attachment.Status != "ready" { + return common.Conflict("attachment is not ready") + } + + ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout) + + body, size, err := h.objects.Get(ctx, attachment.ObjectKey) + if err != nil { + cancel() + if errors.Is(err, objectstore.ErrNotFound) { + slog.Error("attachment object missing", "attachment_id", attachment.ID) + return common.NotFound("attachment content not found") + } + slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID) + return common.Unavailable("object storage unavailable") + } + if size != attachment.SizeBytes { + _ = body.Close() + cancel() + slog.Error("attachment object size mismatch", + "attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size) + return common.Unavailable("attachment content is inconsistent") + } + + c.Set(fiber.HeaderContentType, attachment.ContentType) + c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10)) + c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename})) + c.Set(fiber.HeaderCacheControl, "private, no-store") + c.Set("X-Content-Type-Options", "nosniff") + + // AGENT: SendStreamWriter copies the S3 body straight to the response. The + // API never materializes the whole object in RAM and never redirects the + // client to a bucket URL, so authorization is enforced on every byte. + // + // The context and body are released inside the callback, not by a defer in + // this handler: the handler returns before fasthttp copies the stream, and + // canceling the S3 request context earlier truncates the response. + return c.SendStreamWriter(func(writer *bufio.Writer) { + defer cancel() + defer func() { _ = body.Close() }() + if _, copyErr := io.Copy(writer, body); copyErr != nil { + slog.Error("stream attachment to client", "err", copyErr, "attachment_id", attachment.ID) + } + }) +} + +func requestBody(c fiber.Ctx) io.Reader { + if reader := c.Request().BodyStream(); reader != nil { + return reader + } + return bytes.NewReader(c.Body()) +} + +func objectKey(chatID, attachmentID uuid.UUID) string { + return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID) +} + +// CleanupStaleUploads removes temp files abandoned by a previous crash. A +// running request removes its own file in a defer; this only handles the +// process-died-mid-upload case. +func CleanupStaleUploads(dir string, olderThan time.Duration) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + cutoff := time.Now().Add(-olderThan) + for _, entry := range entries { + if entry.IsDir() { + continue + } + info, infoErr := entry.Info() + if infoErr != nil || info.ModTime().After(cutoff) { + continue + } + _ = os.Remove(dir + string(os.PathSeparator) + entry.Name()) + } +} diff --git a/app/internal/attachments/image.go b/app/internal/attachments/image.go new file mode 100644 index 0000000..cff74a8 --- /dev/null +++ b/app/internal/attachments/image.go @@ -0,0 +1,129 @@ +// Package attachments implements the synchronous upload, metadata lookup and +// authorized download path for chat attachments. +package attachments + +import ( + "context" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "net/http" + "os" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "golang.org/x/sync/semaphore" +) + +type Validator struct { + maxPixels int64 + decodeBudget *semaphore.Weighted + budgetBytes int64 +} + +type ImageInfo struct { + ContentType string + Width int + Height int +} + +func NewValidator(maxPixels, decodeBudgetBytes int64) *Validator { + return &Validator{ + maxPixels: maxPixels, + decodeBudget: semaphore.NewWeighted(decodeBudgetBytes), + budgetBytes: decodeBudgetBytes, + } +} + +// Validate checks the declared media type against the real bytes and then +// decodes the image under a shared memory budget. The decode is what turns a +// structurally broken or truncated file into 415 instead of a saved object. +// +// AGENT: image.DecodeConfig first reads only the header. This is the important +// ordering: a decompression bomb is rejected from width×height before the +// decoder is allowed to allocate pixel memory. +func (v *Validator) Validate(ctx context.Context, file *os.File, declared string) (ImageInfo, error) { + mediaType, _, err := mime.ParseMediaType(declared) + if err != nil { + return ImageInfo{}, common.UnsupportedMedia("Content-Type must be image/png or image/jpeg") + } + if mediaType != "image/png" && mediaType != "image/jpeg" { + return ImageInfo{}, common.UnsupportedMedia("Content-Type must be image/png or image/jpeg") + } + + if _, err = file.Seek(0, 0); err != nil { + return ImageInfo{}, common.BadRequest("cannot read upload body") + } + head := make([]byte, 512) + n, readErr := file.Read(head) + if readErr != nil && !errors.Is(readErr, io.EOF) { + return ImageInfo{}, common.BadRequest("cannot read upload body") + } + detected := http.DetectContentType(head[:n]) + if detected != mediaType { + return ImageInfo{}, common.UnsupportedMedia("file content does not match Content-Type") + } + + if _, err = file.Seek(0, 0); err != nil { + return ImageInfo{}, common.BadRequest("cannot read upload body") + } + config, format, err := image.DecodeConfig(file) + if err != nil { + return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image") + } + if format != "png" && format != "jpeg" { + return ImageInfo{}, common.UnsupportedMedia("only PNG and JPEG images are accepted") + } + if (format == "png" && mediaType != "image/png") || (format == "jpeg" && mediaType != "image/jpeg") { + return ImageInfo{}, common.UnsupportedMedia("file content does not match Content-Type") + } + if config.Width <= 0 || config.Height <= 0 { + return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image") + } + + pixels := int64(config.Width) * int64(config.Height) + if pixels <= 0 || pixels > v.maxPixels { + return ImageInfo{}, common.PayloadTooLarge("image has too many pixels") + } + + // Approximate decoded RGBA size. The semaphore caps the sum across + // concurrent requests; a single 40 Mp image needs ~160 MiB, so a lower + // configured budget is a deliberate operator choice and becomes a 413. + weight := pixels * 4 + if weight > v.budgetBytes { + return ImageInfo{}, common.PayloadTooLarge("image is too large to validate") + } + if err = v.decodeBudget.Acquire(ctx, weight); err != nil { + return ImageInfo{}, common.Unavailable("image validation is busy") + } + defer v.decodeBudget.Release(weight) + + if _, err = file.Seek(0, 0); err != nil { + return ImageInfo{}, common.BadRequest("cannot read upload body") + } + info, decodeErr := safeDecode(file) + if decodeErr != nil { + return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image") + } + bounds := info.Bounds() + if bounds.Dx() != config.Width || bounds.Dy() != config.Height { + return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image") + } + + return ImageInfo{ContentType: mediaType, Width: config.Width, Height: config.Height}, nil +} + +// safeDecode converts a decoder panic into an ordinary error. A malformed +// client file must never crash the process or become a 500. +func safeDecode(file *os.File) (img image.Image, err error) { + defer func() { + if recovered := recover(); recovered != nil { + img = nil + err = errors.New("image decoder panic") + } + }() + img, _, err = image.Decode(file) + return img, err +} diff --git a/app/internal/attachments/image_test.go b/app/internal/attachments/image_test.go new file mode 100644 index 0000000..75030b5 --- /dev/null +++ b/app/internal/attachments/image_test.go @@ -0,0 +1,134 @@ +package attachments + +import ( + "bytes" + "context" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "path/filepath" + "testing" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" +) + +// writeTemp stores raw bytes in a temporary file and returns its path. The +// validator works on *os.File so it can seek, sniff and decode the same body +// without loading it into memory twice. +func writeTemp(t *testing.T, name string, data []byte) *os.File { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + t.Cleanup(func() { _ = file.Close() }) + return file +} + +func encodePNG(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for x := 0; x < width; x++ { + for y := 0; y < height; y++ { + img.Set(x, y, color.RGBA{R: uint8(x), G: uint8(y), B: 0x40, A: 0xff}) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png: %v", err) + } + return buf.Bytes() +} + +func TestValidateAcceptsPNGAndJPEG(t *testing.T) { + validator := NewValidator(40_000_000, 256<<20) + ctx := context.Background() + + pngFile := writeTemp(t, "image.png", encodePNG(t, 8, 6)) + info, err := validator.Validate(ctx, pngFile, "image/png") + if err != nil { + t.Fatalf("valid png rejected: %v", err) + } + if info.ContentType != "image/png" || info.Width != 8 || info.Height != 6 { + t.Fatalf("unexpected png info: %+v", info) + } + + img := image.NewRGBA(image.Rect(0, 0, 4, 3)) + var buf bytes.Buffer + if err = jpeg.Encode(&buf, img, nil); err != nil { + t.Fatalf("encode jpeg: %v", err) + } + jpegFile := writeTemp(t, "image.jpg", buf.Bytes()) + info, err = validator.Validate(ctx, jpegFile, "image/jpeg; charset=binary") + if err != nil { + t.Fatalf("valid jpeg rejected: %v", err) + } + if info.ContentType != "image/jpeg" || info.Width != 4 || info.Height != 3 { + t.Fatalf("unexpected jpeg info: %+v", info) + } +} + +func TestValidateRejectsMismatchedOrBrokenContent(t *testing.T) { + validator := NewValidator(40_000_000, 256<<20) + ctx := context.Background() + pngBytes := encodePNG(t, 8, 6) + + cases := []struct { + name string + data []byte + declared string + code string + }{ + {"png declared as jpeg", pngBytes, "image/jpeg", common.ErrorCodeUnsupportedMedia}, + {"text declared as png", []byte("this is not an image at all"), "image/png", common.ErrorCodeUnsupportedMedia}, + {"unsupported declared type", pngBytes, "image/gif", common.ErrorCodeUnsupportedMedia}, + {"garbage declared type", pngBytes, "not a media type", common.ErrorCodeUnsupportedMedia}, + {"truncated png", pngBytes[:len(pngBytes)/2], "image/png", common.ErrorCodeUnsupportedMedia}, + {"empty body", nil, "image/png", common.ErrorCodeUnsupportedMedia}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + file := writeTemp(t, "upload", tc.data) + if _, err := validator.Validate(ctx, file, tc.declared); !common.IsCode(err, tc.code) { + t.Fatalf("Validate(%q) = %v, want code %s", tc.name, err, tc.code) + } + }) + } +} + +func TestValidateRejectsTooManyPixelsBeforeDecoding(t *testing.T) { + validator := NewValidator(100, 256<<20) + file := writeTemp(t, "big.png", encodePNG(t, 20, 20)) + + _, err := validator.Validate(context.Background(), file, "image/png") + if !common.IsCode(err, common.ErrorCodePayloadTooLarge) { + t.Fatalf("Validate = %v, want %s", err, common.ErrorCodePayloadTooLarge) + } +} + +func TestValidateRejectsImageAboveDecodeBudget(t *testing.T) { + // A 100-pixel image needs ~400 bytes of RGBA memory, more than this budget. + validator := NewValidator(40_000_000, 100) + file := writeTemp(t, "budget.png", encodePNG(t, 10, 10)) + + _, err := validator.Validate(context.Background(), file, "image/png") + if !common.IsCode(err, common.ErrorCodePayloadTooLarge) { + t.Fatalf("Validate = %v, want %s", err, common.ErrorCodePayloadTooLarge) + } +} + +func TestSafeDecodeConvertsPanicToError(t *testing.T) { + // image.Decode does not panic on arbitrary bytes, so this only guards the + // contract that safeDecode always returns an error instead of crashing. + file := writeTemp(t, "junk", []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0xff}) + if _, err := safeDecode(file); err == nil { + t.Fatal("safeDecode accepted junk bytes") + } +} diff --git a/app/internal/chats/handler.go b/app/internal/chats/handler.go index a791fdf..b6cb7a0 100644 --- a/app/internal/chats/handler.go +++ b/app/internal/chats/handler.go @@ -9,11 +9,12 @@ import ( ) type Handler struct { - store *store.Store + store *store.Store + jsonBodyLimit int64 } -func NewHandler(st *store.Store) *Handler { - return &Handler{store: st} +func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler { + return &Handler{store: st, jsonBodyLimit: jsonBodyLimit} } type CreateChatRequest struct { @@ -25,7 +26,7 @@ func (h *Handler) CreateChat(c fiber.Ctx) error { userID := common.MustUserID(c) var req CreateChatRequest - if err := common.BindJSON(c, &req); err != nil { + if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil { return err } diff --git a/app/internal/common/attachments_validation_test.go b/app/internal/common/attachments_validation_test.go new file mode 100644 index 0000000..0387112 --- /dev/null +++ b/app/internal/common/attachments_validation_test.go @@ -0,0 +1,64 @@ +package common + +import ( + "strings" + "testing" + + "github.com/google/uuid" +) + +func TestValidateFilename(t *testing.T) { + valid := []string{"photo.png", "фото 1.jpeg", "a", strings.Repeat("a", 255)} + for _, name := range valid { + if err := ValidateFilename(name); err != nil { + t.Errorf("ValidateFilename(%q) = %v, want nil", name, err) + } + } + + invalid := []string{ + "", " ", ".", "..", "dir/photo.png", `dir\photo.png`, + "nul\x00byte", "line\nbreak", strings.Repeat("a", 256), + } + for _, name := range invalid { + if err := ValidateFilename(name); err == nil { + t.Errorf("ValidateFilename(%q) = nil, want error", name) + } + } +} + +func TestValidateAttachmentIDs(t *testing.T) { + ids := make([]uuid.UUID, 10) + for index := range ids { + ids[index] = uuid.New() + } + if err := ValidateAttachmentIDs(ids); err != nil { + t.Fatalf("ten unique ids rejected: %v", err) + } + if err := ValidateAttachmentIDs(append(ids, uuid.New())); err == nil { + t.Fatal("eleven ids accepted") + } + if err := ValidateAttachmentIDs([]uuid.UUID{ids[0], ids[0]}); err == nil { + t.Fatal("duplicate ids accepted") + } + if err := ValidateAttachmentIDs(nil); err != nil { + t.Fatalf("empty list rejected: %v", err) + } +} + +func TestValidateMessageText(t *testing.T) { + if err := ValidateMessageText("", 1); err != nil { + t.Fatalf("empty text with attachment rejected: %v", err) + } + if err := ValidateMessageText(" ", 1); err != nil { + t.Fatalf("blank text with attachment rejected: %v", err) + } + if err := ValidateMessageText("", 0); err == nil { + t.Fatal("empty text without attachment accepted") + } + if err := ValidateMessageText(strings.Repeat("ж", 4000), 0); err != nil { + t.Fatalf("4000 code points rejected: %v", err) + } + if err := ValidateMessageText(strings.Repeat("ж", 4001), 1); err == nil { + t.Fatal("4001 code points accepted") + } +} diff --git a/app/internal/common/binding.go b/app/internal/common/binding.go index cc30fa3..bba6208 100644 --- a/app/internal/common/binding.go +++ b/app/internal/common/binding.go @@ -9,12 +9,40 @@ import ( "github.com/gofiber/fiber/v3" ) -func BindJSON(c fiber.Ctx, dst any) error { +// ReadBodyLimited reads at most limit bytes from the request body. The caller +// must use it instead of c.Body() when the endpoint has a smaller budget than +// the process-wide Fiber.BodyLimit. The limit itself belongs to the caller +// (config.MaxJSONBodyBytes for JSON endpoints). +// +// with StreamRequestBody enabled fasthttp exposes the network body as a +// stream, so io.ReadAll over a LimitReader allocates only the bytes the +// endpoint actually accepts instead of the whole upload limit. +func ReadBodyLimited(c fiber.Ctx, limit int64) ([]byte, error) { + var reader io.Reader = c.Request().BodyStream() + if reader == nil { + reader = bytes.NewReader(c.Body()) + } + + data, err := io.ReadAll(io.LimitReader(reader, limit+1)) + if err != nil { + return nil, BadRequest("incomplete request body") + } + if int64(len(data)) > limit { + return nil, PayloadTooLarge("request body is too large") + } + + return data, nil +} + +func BindJSON(c fiber.Ctx, dst any, limit int64) error { if contentType := c.Get(fiber.HeaderContentType); !strings.HasPrefix(contentType, "application/json") { return UnsupportedMedia("invalid JSON body") } - body := c.Body() + body, err := ReadBodyLimited(c, limit) + if err != nil { + return err + } if len(body) == 0 { return BadRequest("request body is empty") } diff --git a/app/internal/common/error.go b/app/internal/common/error.go index 817bad0..899dde6 100644 --- a/app/internal/common/error.go +++ b/app/internal/common/error.go @@ -79,6 +79,10 @@ func UnsupportedMedia(message string) *AppError { return NewAppError(http.StatusUnsupportedMediaType, ErrorCodeUnsupportedMedia, message) } +func PayloadTooLarge(message string) *AppError { + return NewAppError(http.StatusRequestEntityTooLarge, ErrorCodePayloadTooLarge, message) +} + func Internal() *AppError { return NewAppError(http.StatusInternalServerError, ErrorCodeInternal, "Internal server error") } diff --git a/app/internal/common/validation.go b/app/internal/common/validation.go index eb20013..4cf9ab4 100644 --- a/app/internal/common/validation.go +++ b/app/internal/common/validation.go @@ -3,6 +3,7 @@ package common import ( "regexp" "strings" + "unicode" "unicode/utf8" "github.com/google/uuid" @@ -62,13 +63,49 @@ func ValidateChatTitle(title string) error { return nil } -func ValidateMessageText(text string) error { - if blank(text) { +func ValidateMessageText(text string, attachmentCount int) error { + if attachmentCount == 0 && blank(text) { return BadRequest("text must not be empty or only whitespace") } - if !strLenBetween(text, 1, 4000) { - return BadRequest("text must be between 1 and 4000 Unicode code points") + if !strLenBetween(text, 0, 4000) { + return BadRequest("text must be between 0 and 4000 Unicode code points") + } + + return nil +} + +func ValidateAttachmentIDs(ids []uuid.UUID) error { + if len(ids) > 10 { + return BadRequest("attachment_ids must contain at most 10 items") + } + + seen := make(map[uuid.UUID]bool, len(ids)) + for _, id := range ids { + if seen[id] { + return BadRequest("attachment_ids must not contain duplicates") + } + seen[id] = true + } + + return nil +} + +func ValidateFilename(filename string) error { + if !strLenBetween(filename, 1, 255) { + return BadRequest("filename must be between 1 and 255 Unicode code points") + } + if blank(filename) { + return BadRequest("filename must not be empty or only whitespace") + } + if filename == "." || filename == ".." { + return BadRequest("filename is not allowed") + } + + for _, r := range filename { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return BadRequest("filename contains a forbidden character") + } } return nil diff --git a/app/internal/config/config.go b/app/internal/config/config.go index 70d2d8a..7b4aa12 100644 --- a/app/internal/config/config.go +++ b/app/internal/config/config.go @@ -5,11 +5,23 @@ package config import ( "fmt" "os" + "path/filepath" "strconv" "strings" "time" ) +const ( + // Configuration may make a limit stricter, never looser. + MaxUploadBytesHardCap = 10 * 1024 * 1024 + MaxImagePixelsHardCap = 40_000_000 + + // MaxJSONBodyBytes is the body budget of every JSON endpoint. It is also + // the slack between the global HTTP body limit and the upload limit, so a + // smaller MAX_UPLOAD_BYTES cannot strangle JSON requests. + MaxJSONBodyBytes = 1 * 1024 * 1024 +) + type Config struct { Host string Port string @@ -24,6 +36,30 @@ type Config struct { StartupWait time.Duration RunMigrations bool MigrateOnly bool + + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + UploadTimeout time.Duration + DownloadTimeout time.Duration + + MaxUploadBytes int64 + MaxImagePixels int64 + ImageDecodeBytes int64 + UploadTempDir string + + // HTTP timeouts and body limits are derived once in Load so that app.go + // and the handlers never invent values of their own. + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + UploadDrainLimit int64 + HTTPBodyLimit int + + ReconcileGrace time.Duration + ReconcileInterval time.Duration } func Load() (Config, error) { @@ -45,17 +81,42 @@ func Load() (Config, error) { return Config{}, err } - cfg := Config{ - Host: envOr("APP_HOST", "0.0.0.0"), - Port: envOr("APP_PORT", "8080"), - InstanceID: instance, - SessionTTL: time.Duration(sessionTTL) * time.Second, - IdempotencyTTL: time.Duration(idempotencyTTL) * time.Second, - DBMaxConns: int32(maxConns), - DBTimeout: 5 * time.Second, - RedisTimeout: 2 * time.Second, - StartupWait: 30 * time.Second, + maxUpload, err := envInt[int64]("MAX_UPLOAD_BYTES", MaxUploadBytesHardCap, 1, MaxUploadBytesHardCap) + if err != nil { + return Config{}, err } + maxPixels, err := envInt[int64]("MAX_IMAGE_PIXELS", MaxImagePixelsHardCap, 1, MaxImagePixelsHardCap) + if err != nil { + return Config{}, err + } + decodeMB, err := envInt("IMAGE_DECODE_MEMORY_MB", 256, 32, 4096) + if err != nil { + return Config{}, err + } + + cfg := Config{ + Host: envOr("APP_HOST", "0.0.0.0"), + Port: envOr("APP_PORT", "8080"), + InstanceID: instance, + SessionTTL: time.Duration(sessionTTL) * time.Second, + IdempotencyTTL: time.Duration(idempotencyTTL) * time.Second, + DBMaxConns: int32(maxConns), + DBTimeout: 5 * time.Second, + RedisTimeout: 2 * time.Second, + StartupWait: 30 * time.Second, + S3Region: envOr("S3_REGION", "us-east-1"), + S3Bucket: envOr("S3_BUCKET", "messenger-attachments"), + ReadTimeout: 30 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 60 * time.Second, + UploadTimeout: 30 * time.Second, + DownloadTimeout: 60 * time.Second, + MaxUploadBytes: maxUpload, + MaxImagePixels: maxPixels, + ImageDecodeBytes: int64(decodeMB) * 1024 * 1024, + UploadTempDir: envOr("UPLOAD_TEMP_DIR", filepath.Join(os.TempDir(), "messenger-uploads")), + } + cfg.deriveHTTPLimits() if cfg.RunMigrations, err = envBool("RUN_MIGRATIONS", true); err != nil { return Config{}, err @@ -63,6 +124,22 @@ func Load() (Config, error) { if cfg.MigrateOnly, err = envBool("MIGRATE_ONLY", false); err != nil { return Config{}, err } + reconcileOnly, err := envBool("RECONCILE_ONLY", false) + if err != nil { + return Config{}, err + } + + reconcileGrace, err := envInt("RECONCILE_GRACE_SECONDS", 900, 60, 0) + if err != nil { + return Config{}, err + } + cfg.ReconcileGrace = time.Duration(reconcileGrace) * time.Second + + reconcileInterval, err := envInt("RECONCILE_INTERVAL_SECONDS", 300, 10, 0) + if err != nil { + return Config{}, err + } + cfg.ReconcileInterval = time.Duration(reconcileInterval) * time.Second if cfg.DatabaseURL, err = secret("DATABASE_URL"); err != nil { return Config{}, err @@ -75,6 +152,29 @@ func Load() (Config, error) { return cfg, nil } + if cfg.S3Endpoint, err = secret("S3_ENDPOINT"); err != nil { + return Config{}, err + } + if cfg.S3Endpoint == "" { + return Config{}, fmt.Errorf("S3_ENDPOINT or S3_ENDPOINT_FILE is required") + } + if cfg.S3AccessKey, err = secret("S3_ACCESS_KEY_ID"); err != nil { + return Config{}, err + } + if cfg.S3AccessKey == "" { + return Config{}, fmt.Errorf("S3_ACCESS_KEY_ID or S3_ACCESS_KEY_ID_FILE is required") + } + if cfg.S3SecretKey, err = secret("S3_SECRET_ACCESS_KEY"); err != nil { + return Config{}, err + } + if cfg.S3SecretKey == "" { + return Config{}, fmt.Errorf("S3_SECRET_ACCESS_KEY or S3_SECRET_ACCESS_KEY_FILE is required") + } + + if reconcileOnly { + return cfg, nil + } + if cfg.CacheURL, err = secret("CACHE_URL"); err != nil { return Config{}, err } @@ -94,6 +194,15 @@ func Load() (Config, error) { return cfg, nil } +// deriveHTTPLimits is the single place that relates the three HTTP budgets: +// the transport gate must admit both a full upload and a full JSON body, and it +// must leave room for the handler to drain an oversized declared body so the +// client receives a 413 instead of a connection reset. +func (c *Config) deriveHTTPLimits() { + c.UploadDrainLimit = c.MaxUploadBytes + MaxJSONBodyBytes + c.HTTPBodyLimit = int(c.UploadDrainLimit) +} + func instanceID() (string, error) { if value := strings.TrimSpace(os.Getenv("INSTANCE_ID")); value != "" { return value, nil @@ -120,21 +229,27 @@ func envBool(name string, fallback bool) (bool, error) { return parsed, nil } -// envInt parses an integer setting; max <= 0 means "no upper bound". -func envInt(name string, fallback, min, max int) (int, error) { +// envInt parses an integer setting into int or int64; max <= 0 means "no upper +// bound". Both widths are needed by Config, and one helper keeps validation +// identical for every numeric variable. +func envInt[T ~int | ~int64](name string, fallback, min, max T) (T, error) { value := os.Getenv(name) if value == "" { return fallback, nil } - parsed, err := strconv.Atoi(value) - if err != nil || parsed < min || (max > 0 && parsed > max) { + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < int64(min) || (max > 0 && parsed > int64(max)) { if max > 0 { return 0, fmt.Errorf("%s must be an integer between %d and %d", name, min, max) } return 0, fmt.Errorf("%s must be an integer >= %d", name, min) } - return parsed, nil + result := T(parsed) + if int64(result) != parsed { + return 0, fmt.Errorf("%s is out of range", name) + } + return result, nil } func envOr(name, fallback string) string { diff --git a/app/internal/config/config_test.go b/app/internal/config/config_test.go index 5818680..b56546b 100644 --- a/app/internal/config/config_test.go +++ b/app/internal/config/config_test.go @@ -25,3 +25,40 @@ func TestInstanceIDFallsBackToHostname(t *testing.T) { t.Fatal("instanceID returned an empty value") } } + +func TestDerivedHTTPLimits(t *testing.T) { + cfg := Config{MaxUploadBytes: 2 * 1024 * 1024} + cfg.deriveHTTPLimits() + + if cfg.UploadDrainLimit != cfg.MaxUploadBytes+MaxJSONBodyBytes { + t.Fatalf("drain limit = %d, want upload + JSON body", cfg.UploadDrainLimit) + } + if cfg.HTTPBodyLimit != int(cfg.UploadDrainLimit) { + t.Fatalf("body limit = %d, want %d", cfg.HTTPBodyLimit, cfg.UploadDrainLimit) + } + if cfg.HTTPBodyLimit-int(cfg.MaxUploadBytes) != MaxJSONBodyBytes { + t.Fatal("transport gate would strangle a full JSON body") + } +} + +func TestEnvIntHandlesBothWidths(t *testing.T) { + t.Setenv("TEST_ENV_INT", "42") + + small, err := envInt("TEST_ENV_INT", 1, 1, 100) + if err != nil || small != 42 { + t.Fatalf("envInt = %d, %v; want 42, nil", small, err) + } + large, err := envInt[int64]("TEST_ENV_INT", 1, 1, 100) + if err != nil || large != 42 { + t.Fatalf("envInt[int64] = %d, %v; want 42, nil", large, err) + } +} + +func TestEnvIntRejectsOutOfRange(t *testing.T) { + for _, value := range []string{"0", "101", "not-a-number"} { + t.Setenv("TEST_ENV_INT", value) + if _, err := envInt("TEST_ENV_INT", 1, 1, 100); err == nil { + t.Fatalf("envInt accepted %q", value) + } + } +} diff --git a/app/internal/database/migrations/0003_attachments.down.sql b/app/internal/database/migrations/0003_attachments.down.sql new file mode 100644 index 0000000..76f33cd --- /dev/null +++ b/app/internal/database/migrations/0003_attachments.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS message_attachments; +DROP TABLE IF EXISTS attachments; + +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_len; +ALTER TABLE messages + ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000); diff --git a/app/internal/database/migrations/0003_attachments.up.sql b/app/internal/database/migrations/0003_attachments.up.sql new file mode 100644 index 0000000..badf223 --- /dev/null +++ b/app/internal/database/migrations/0003_attachments.up.sql @@ -0,0 +1,46 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_len; +ALTER TABLE messages + ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 bytea NOT NULL, + object_key text NOT NULL, + status text NOT NULL, + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'failed')), + CONSTRAINT attachments_error_state CHECK ( + (status = 'pending' AND error_code IS NULL) OR + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) + ), + CONSTRAINT attachments_object_key_key UNIQUE (object_key) +); + +CREATE INDEX IF NOT EXISTS attachments_chat_created_idx + ON attachments (chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS attachments_status_created_idx + ON attachments (status, created_at); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position >= 0), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx + ON message_attachments (attachment_id); diff --git a/app/internal/database/migrations_test.go b/app/internal/database/migrations_test.go index a40aeb2..bd1812b 100644 --- a/app/internal/database/migrations_test.go +++ b/app/internal/database/migrations_test.go @@ -3,27 +3,59 @@ package database import ( "errors" "io/fs" + "regexp" + "strconv" "testing" "github.com/golang-migrate/migrate/v4/source/iofs" ) -func TestEmbeddedMigrationsParse(t *testing.T) { +// latestMigrationVersion is the only value to bump when a migration is added. +// The rest of the test derives filenames and ordering from the directory. +const latestMigrationVersion = 3 + +var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`) + +func TestEmbeddedMigrationsAreCompleteAndContiguous(t *testing.T) { entries, err := fs.ReadDir(migrationFiles, "migrations") if err != nil { t.Fatalf("read embedded migrations: %v", err) } - names := make(map[string]bool, len(entries)) + up := make(map[int]string, latestMigrationVersion) + down := make(map[int]string, latestMigrationVersion) + for _, entry := range entries { - names[entry.Name()] = true + if entry.IsDir() { + continue + } + name := entry.Name() + match := migrationFilePattern.FindStringSubmatch(name) + if match == nil { + t.Fatalf("migration %q does not match NNNN_name.{up,down}.sql", name) + } + + version, parseErr := strconv.Atoi(match[1]) + if parseErr != nil || version < 1 || version > latestMigrationVersion { + t.Fatalf("migration %q has version outside 1..%d", name, latestMigrationVersion) + } + + files := up + if match[2] == "down" { + files = down + } + if previous, ok := files[version]; ok { + t.Fatalf("version %d has two %s files: %q and %q", version, match[2], previous, name) + } + files[version] = name } - for _, want := range []string{ - "0001_init.up.sql", "0001_init.down.sql", - "0002_message_idempotency.up.sql", "0002_message_idempotency.down.sql", - } { - if !names[want] { - t.Fatalf("migration %q is missing from the embedded filesystem", want) + + for version := 1; version <= latestMigrationVersion; version++ { + if up[version] == "" { + t.Fatalf("version %d is missing an up migration", version) + } + if down[version] == "" { + t.Fatalf("version %d is missing a down migration", version) } } @@ -37,17 +69,20 @@ func TestEmbeddedMigrationsParse(t *testing.T) { if err != nil { t.Fatalf("read first migration: %v", err) } - if version != 1 { - t.Fatalf("first migration version = %d, want 1", version) - } - version, err = source.Next(version) - if err != nil { - t.Fatalf("read second migration: %v", err) - } - if version != 2 { - t.Fatalf("second migration version = %d, want 2", version) - } - if _, err := source.Next(version); !errors.Is(err, fs.ErrNotExist) { - t.Fatalf("expected 0002 to be the last migration, got %v", err) + for expected := 1; expected <= latestMigrationVersion; expected++ { + if version != uint(expected) { + t.Fatalf("migration order = %d, want %d", version, expected) + } + next, nextErr := source.Next(version) + if expected == latestMigrationVersion { + if !errors.Is(nextErr, fs.ErrNotExist) { + t.Fatalf("expected %d to be the last migration, got %v", version, nextErr) + } + break + } + if nextErr != nil { + t.Fatalf("read migration after %d: %v", version, nextErr) + } + version = next } } diff --git a/app/internal/health/handler.go b/app/internal/health/handler.go index 79f8958..27c5088 100644 --- a/app/internal/health/handler.go +++ b/app/internal/health/handler.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/session" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" "github.com/gofiber/fiber/v3" @@ -14,10 +15,11 @@ import ( type Handler struct { store *store.Store sessions *session.Store + objects *objectstore.Client } -func NewHandler(st *store.Store, sessions *session.Store) *Handler { - return &Handler{store: st, sessions: sessions} +func NewHandler(st *store.Store, sessions *session.Store, objects *objectstore.Client) *Handler { + return &Handler{store: st, sessions: sessions, objects: objects} } var shuttingDown atomic.Bool @@ -50,5 +52,10 @@ func (h *Handler) Ready(c fiber.Ctx) error { common.ErrorCodeUnavailable, "session store unavailable") } + if err := h.objects.Ping(c.Context()); err != nil { + return common.SendError(c, fiber.StatusServiceUnavailable, + common.ErrorCodeUnavailable, "object storage unavailable") + } + return c.JSON(Status{Status: "ok"}) } diff --git a/app/internal/messages/handler.go b/app/internal/messages/handler.go index 470591e..7194da7 100644 --- a/app/internal/messages/handler.go +++ b/app/internal/messages/handler.go @@ -6,18 +6,21 @@ import ( "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" "github.com/gofiber/fiber/v3" + "github.com/google/uuid" ) type Handler struct { - store *store.Store + store *store.Store + jsonBodyLimit int64 } -func NewHandler(st *store.Store) *Handler { - return &Handler{store: st} +func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler { + return &Handler{store: st, jsonBodyLimit: jsonBodyLimit} } type CreateMessageRequest struct { - Text string `json:"text"` + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` } func (h *Handler) CreateMessage(c fiber.Ctx) error { @@ -34,14 +37,17 @@ func (h *Handler) CreateMessage(c fiber.Ctx) error { } var req CreateMessageRequest - if err := common.BindJSON(c, &req); err != nil { + if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil { return err } - if err := common.ValidateMessageText(req.Text); err != nil { + if err := common.ValidateAttachmentIDs(req.AttachmentIDs); err != nil { + return err + } + if err := common.ValidateMessageText(req.Text, len(req.AttachmentIDs)); err != nil { return err } - message, err := h.store.CreateMessage(c.Context(), chatID, userID, req.Text, idempotencyKey) + message, err := h.store.CreateMessage(c.Context(), chatID, userID, req.Text, idempotencyKey, req.AttachmentIDs) if err != nil { return err } diff --git a/app/internal/model/model.go b/app/internal/model/model.go index 07f1e71..b49abdc 100644 --- a/app/internal/model/model.go +++ b/app/internal/model/model.go @@ -31,16 +31,47 @@ func (c Chat) GetID() uuid.UUID { return c.ID } func (c Chat) GetCreatedAt() time.Time { return time.Time(c.CreatedAt) } type Message struct { - ID uuid.UUID `json:"id"` - ChatID uuid.UUID `json:"chat_id"` - SenderID uuid.UUID `json:"sender_id"` - Text string `json:"text"` - CreatedAt common.RFCFormTime `json:"created_at"` + ID uuid.UUID `json:"id"` + ChatID uuid.UUID `json:"chat_id"` + SenderID uuid.UUID `json:"sender_id"` + Text string `json:"text"` + CreatedAt common.RFCFormTime `json:"created_at"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` } func (m Message) GetID() uuid.UUID { return m.ID } func (m Message) GetCreatedAt() time.Time { return time.Time(m.CreatedAt) } +type Attachment struct { + ID uuid.UUID `json:"id"` + ChatID uuid.UUID `json:"chat_id"` + OwnerID uuid.UUID `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt common.RFCFormTime `json:"created_at"` + UpdatedAt common.RFCFormTime `json:"updated_at"` + + // ObjectKey is internal and never serialized. + ObjectKey string `json:"-"` +} + +func (a Attachment) GetID() uuid.UUID { return a.ID } +func (a Attachment) GetCreatedAt() time.Time { return time.Time(a.CreatedAt) } + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + type Session struct { Token string `json:"token"` TokenType string `json:"token_type"` diff --git a/app/internal/objectstore/client.go b/app/internal/objectstore/client.go new file mode 100644 index 0000000..3cb8ea2 --- /dev/null +++ b/app/internal/objectstore/client.go @@ -0,0 +1,176 @@ +// Package objectstore is the only place that knows how to talk to the private S3 +// bucket. PostgreSQL remains the source of truth for metadata; this package +// stores and returns opaque bytes only. +package objectstore + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +var ErrNotFound = errors.New("object not found") + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string +} + +type Client struct { + s3 *s3.Client + bucket string +} + +// Object describes one key returned by List. It intentionally exposes only the +// fields the reconciler needs. +type Object struct { + Key string + Size int64 + LastModified time.Time +} + +func New(cfg Config) (*Client, error) { + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfg.AccessKey, cfg.SecretKey, "")), + ) + if err != nil { + return nil, err + } + + // AGENT: a bounded retry count keeps a broken S3 dependency from turning an + // HTTP upload into an unbounded wait. Context deadlines in the handlers set + // the overall request budget. + awsCfg.RetryMaxAttempts = 2 + + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = 20 + transport.MaxIdleConns = 40 + + client := s3.NewFromConfig(awsCfg, func(options *s3.Options) { + // RustFS is addressed by its Compose service name, so virtual-host + // addressing cannot work without extra DNS entries. Path-style is the + // portable choice for every S3-compatible server used in this stand. + options.UsePathStyle = true + options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/")) + options.HTTPClient = &http.Client{Transport: transport} + }) + + return &Client{s3: client, bucket: cfg.Bucket}, nil +} + +func (c *Client) Bucket() string { return c.bucket } + +// Ping is used by /health/ready. HeadBucket is the cheapest authenticated call +// that proves both the endpoint and the scoped credentials work. +func (c *Client) Ping(ctx context.Context) error { + pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + _, err := c.s3.HeadBucket(pingCtx, &s3.HeadBucketInput{Bucket: aws.String(c.bucket)}) + return mapError(err) +} + +func (c *Client) Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error { + _, err := c.s3.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + Body: body, + ContentLength: aws.Int64(size), + ContentType: aws.String(contentType), + }) + return mapError(err) +} + +func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { + output, err := c.s3.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, 0, mapError(err) + } + + size := int64(0) + if output.ContentLength != nil { + size = *output.ContentLength + } + return output.Body, size, nil +} + +func (c *Client) Delete(ctx context.Context, key string) error { + _, err := c.s3.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + return mapError(err) +} + +func (c *Client) List(ctx context.Context, prefix string) ([]Object, error) { + var objects []Object + var continuation *string + + for { + output, err := c.s3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuation, + }) + if err != nil { + return nil, mapError(err) + } + + for _, item := range output.Contents { + if item.Key == nil { + continue + } + object := Object{Key: *item.Key} + if item.Size != nil { + object.Size = *item.Size + } + if item.LastModified != nil { + object.LastModified = *item.LastModified + } + objects = append(objects, object) + } + + if output.IsTruncated == nil || !*output.IsTruncated { + return objects, nil + } + continuation = output.NextContinuationToken + if continuation == nil { + return objects, nil + } + } +} + +// mapError turns a missing object into a sentinel the HTTP layer can map to +// 404, while every other failure stays a dependency error (503). +func mapError(err error) error { + if err == nil { + return nil + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NoSuchBucket", "NotFound", "XMinioInvalidObjectName": + return ErrNotFound + } + } + + return err +} diff --git a/app/internal/reconcile/reconcile.go b/app/internal/reconcile/reconcile.go new file mode 100644 index 0000000..6eabba0 --- /dev/null +++ b/app/internal/reconcile/reconcile.go @@ -0,0 +1,115 @@ +// Package reconcile repairs the two failure windows between PostgreSQL +// metadata and S3 bytes: +// +// - a pending metadata row whose upload never finished or whose 201 was +// never returned; +// - an S3 object whose metadata transaction never committed. +// +// Both repairs use a grace period so an upload that is still in flight is +// never touched. +package reconcile + +import ( + "context" + "errors" + "log/slog" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" +) + +const objectPrefix = "attachments/" + +type Reconciler struct { + store *store.Store + objects *objectstore.Client + grace time.Duration + logger *slog.Logger +} + +type Stats struct { + StalePending int `json:"stale_pending"` + OrphanObjects int `json:"orphan_objects"` + DeletedRows int `json:"deleted_rows"` + DeletedObjects int `json:"deleted_objects"` + Skipped int `json:"skipped"` +} + +func New(st *store.Store, objects *objectstore.Client, grace time.Duration, logger *slog.Logger) *Reconciler { + if logger == nil { + logger = slog.Default() + } + return &Reconciler{store: st, objects: objects, grace: grace, logger: logger} +} + +func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) { + var stats Stats + cutoff := time.Now().UTC().Add(-r.grace) + + // Phase 1: rows that were committed as pending but never reached ready. + stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500) + if err != nil { + return stats, err + } + for _, attachment := range stale { + stats.StalePending++ + if dryRun { + r.logger.Info("reconcile: would remove stale pending attachment", + "attachment_id", attachment.ID, "object_key", attachment.ObjectKey, + "created_at", attachment.CreatedAt) + continue + } + + if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + // Keep the row for the next pass. Deleting it now would lose the + // only pointer to an object that might still exist. + r.logger.Error("reconcile: delete stale object", + "err", err, "attachment_id", attachment.ID) + stats.Skipped++ + continue + } + if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil { + r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID) + stats.Skipped++ + continue + } + stats.DeletedRows++ + stats.DeletedObjects++ + } + + // Phase 2: objects with no metadata row at all. The pending row is written + // before the PUT in the normal path, so this only happens after a crash in + // a narrow window (or manual bucket tampering). + listed, err := r.objects.List(ctx, objectPrefix) + if err != nil { + return stats, err + } + known, err := r.store.ListAttachmentObjectKeys(ctx) + if err != nil { + return stats, err + } + for _, object := range listed { + if _, ok := known[object.Key]; ok { + continue + } + // A missing LastModified must not be treated as ancient: skipping is + // safer than deleting an object that may have just been created. + if object.LastModified.IsZero() || object.LastModified.After(cutoff) { + continue + } + stats.OrphanObjects++ + if dryRun { + r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key) + continue + } + if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) { + r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key) + stats.Skipped++ + continue + } + stats.DeletedObjects++ + } + + return stats, nil +} diff --git a/app/internal/store/attachments.go b/app/internal/store/attachments.go new file mode 100644 index 0000000..cbb85d3 --- /dev/null +++ b/app/internal/store/attachments.go @@ -0,0 +1,170 @@ +package store + +import ( + "context" + "encoding/hex" + "errors" + "time" + + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// CreatePendingAttachment records the intent to store an object before the S3 +// PUT happens. Committing this row first is what makes the reconciler able to +// distinguish "never uploaded" from "uploaded but process died". +func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachment) error { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + hash, err := hex.DecodeString(att.SHA256) + if err != nil { + return common.BadRequest("invalid sha256") + } + + _, err = s.pool.Exec(ctx, ` + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, error_code, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL, $9, $9)`, + att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes, + hash, att.ObjectKey, time.Time(att.CreatedAt)) + return dbError(err) +} + +// MarkAttachmentReady publishes metadata only after the object bytes are known +// to be in S3. A 201 is therefore a promise about both stores. +func (s *Store) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (time.Time, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + updatedAt := time.Now().UTC() + tag, err := s.pool.Exec(ctx, ` + UPDATE attachments + SET status = 'ready', updated_at = $2 + WHERE id = $1 AND status = 'pending'`, id, updatedAt) + if err != nil { + return time.Time{}, dbError(err) + } + if tag.RowsAffected() == 0 { + return time.Time{}, common.Conflict("attachment is not pending") + } + return updatedAt, nil +} + +// DeletePendingAttachment removes a row only while it is still pending. The +// status guard is the safety property: if an upload completed concurrently, the +// compensation or reconciler pass must not delete a ready attachment. +func (s *Store) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + _, err := s.pool.Exec(ctx, + `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id) + return dbError(err) +} + +func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + att, err := scanAttachment(s.pool.QueryRow(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, + a.created_at, a.updated_at, a.object_key + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2 + WHERE a.id = $1`, attachmentID, userID)) + if err != nil { + return model.Attachment{}, err + } + return att, nil +} + +// ListStalePendingAttachments is the entry point for crash recovery. cutoff is +// an instant, not a duration: rows created before it are eligible. The caller +// computes it as now - grace so the boundary can be tuned and tested. +func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, + a.created_at, a.updated_at, a.object_key + FROM attachments a + WHERE a.status = 'pending' AND a.created_at < $1 + ORDER BY a.created_at + LIMIT $2`, cutoff, limit) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + + var result []model.Attachment + for rows.Next() { + att, scanErr := scanAttachment(rows) + if scanErr != nil { + return nil, dbError(scanErr) + } + result = append(result, att) + } + if err = rows.Err(); err != nil { + return nil, dbError(err) + } + return result, nil +} + +func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct{}, error) { + ctx, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query(ctx, `SELECT object_key FROM attachments`) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + + keys := make(map[string]struct{}) + for rows.Next() { + var key string + if err = rows.Scan(&key); err != nil { + return nil, dbError(err) + } + keys[key] = struct{}{} + } + if err = rows.Err(); err != nil { + return nil, dbError(err) + } + return keys, nil +} + +func scanAttachment(row rowScanner) (model.Attachment, error) { + var ( + att model.Attachment + hash []byte + errCode *string + createdAt time.Time + updatedAt time.Time + ) + + if err := row.Scan( + &att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType, + &att.SizeBytes, &hash, &att.Status, &errCode, + &createdAt, &updatedAt, &att.ObjectKey, + ); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, common.NotFound("attachment not found") + } + return model.Attachment{}, dbError(err) + } + + att.SHA256 = hex.EncodeToString(hash) + att.ErrorCode = errCode + att.Variants = []model.Variant{} + att.CreatedAt = common.RFCFormTime(createdAt.UTC()) + att.UpdatedAt = common.RFCFormTime(updatedAt.UTC()) + return att, nil +} diff --git a/app/internal/store/messages.go b/app/internal/store/messages.go index ed395f5..0864d8b 100644 --- a/app/internal/store/messages.go +++ b/app/internal/store/messages.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "errors" + "sort" "time" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" @@ -13,24 +14,29 @@ import ( "github.com/jackc/pgx/v5" ) -func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, text, idempotencyKey string) (model.Message, error) { +func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, text, idempotencyKey string, attachmentIDs []uuid.UUID) (model.Message, error) { ctx, cancel := s.withTimeout(ctx) defer cancel() - fingerprint := sha256.Sum256([]byte(text)) + // Sorting makes the idempotency fingerprint independent of the order + // in which the client listed the same set of attachment IDs. The response + // uses the same deterministic order. + attachmentIDs = canonicalAttachmentIDs(attachmentIDs) + fingerprint := messageFingerprint(text, attachmentIDs) - if message, found, err := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint[:]); err != nil { + if message, found, err := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint); err != nil { return model.Message{}, err } else if found { return message, nil } message := model.Message{ - ID: uuid.New(), - ChatID: chatID, - SenderID: senderID, - Text: text, - CreatedAt: common.NowUTC(), + ID: uuid.New(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: common.NowUTC(), + AttachmentIDs: attachmentIDs, } tx, err := s.pool.Begin(ctx) @@ -39,20 +45,73 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t } defer rollback(tx) - tag, err := tx.Exec(ctx, ` + // Check membership and attachment ownership in the same transaction as the + // insert. A foreign chat or a foreign attachment is intentionally the same + // 404 as a missing object, so the API does not leak existence. + var member int + err = tx.QueryRow(ctx, + `SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2`, + chatID, senderID).Scan(&member) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, common.NotFound("chat not found") + } + if err != nil { + return model.Message{}, dbError(err) + } + + if len(attachmentIDs) > 0 { + rows, queryErr := tx.Query(ctx, ` + SELECT id, status FROM attachments + WHERE chat_id = $1 AND id = ANY($2::uuid[])`, chatID, attachmentIDs) + if queryErr != nil { + return model.Message{}, dbError(queryErr) + } + + found := 0 + for rows.Next() { + var ( + id uuid.UUID + status string + ) + if scanErr := rows.Scan(&id, &status); scanErr != nil { + rows.Close() + return model.Message{}, dbError(scanErr) + } + found++ + if status != "ready" { + rows.Close() + return model.Message{}, common.Conflict("attachment is not ready") + } + } + if err = rows.Err(); err != nil { + rows.Close() + return model.Message{}, dbError(err) + } + rows.Close() + + if found != len(attachmentIDs) { + return model.Message{}, common.NotFound("attachment not found") + } + } + + _, err = tx.Exec(ctx, ` INSERT INTO messages (id, chat_id, sender_id, text, created_at) - SELECT $1, $2, $3, $4, $5 - WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3)`, + VALUES ($1, $2, $3, $4, $5)`, message.ID, message.ChatID, message.SenderID, message.Text, time.Time(message.CreatedAt)) if err != nil { return model.Message{}, dbError(err) } - if tag.RowsAffected() == 0 { - return model.Message{}, common.NotFound("chat not found") + + for position, attachmentID := range attachmentIDs { + if _, err = tx.Exec(ctx, ` + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, $3)`, message.ID, attachmentID, position); err != nil { + return model.Message{}, dbError(err) + } } expiresAt := time.Time(message.CreatedAt).Add(s.idempotencyTTL) - tag, err = tx.Exec(ctx, ` + tag, err := tx.Exec(ctx, ` INSERT INTO message_idempotency (sender_id, chat_id, idempotency_key, request_hash, message_id, created_at, expires_at) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -62,7 +121,7 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t created_at = EXCLUDED.created_at, expires_at = EXCLUDED.expires_at WHERE message_idempotency.expires_at <= now()`, - senderID, chatID, idempotencyKey, fingerprint[:], message.ID, + senderID, chatID, idempotencyKey, fingerprint, message.ID, time.Time(message.CreatedAt), expiresAt) if err != nil { return model.Message{}, dbError(err) @@ -70,7 +129,7 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t if tag.RowsAffected() == 0 { rollback(tx) - stored, found, lookupErr := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint[:]) + stored, found, lookupErr := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint) if lookupErr != nil { return model.Message{}, lookupErr } @@ -87,13 +146,47 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t return message, nil } +func canonicalAttachmentIDs(ids []uuid.UUID) []uuid.UUID { + result := append([]uuid.UUID(nil), ids...) + sort.Slice(result, func(i, j int) bool { + return bytes.Compare(result[i][:], result[j][:]) < 0 + }) + if result == nil { + return []uuid.UUID{} + } + return result +} + +// messageFingerprint keeps the lab3 hash for messages without attachments, so +// an idempotency key stored before this migration still replays correctly. For +// messages with attachments the sorted IDs are part of the canonical body. +func messageFingerprint(text string, attachmentIDs []uuid.UUID) []byte { + if len(attachmentIDs) == 0 { + sum := sha256.Sum256([]byte(text)) + return sum[:] + } + + hash := sha256.New() + hash.Write([]byte(text)) + hash.Write([]byte{0}) + for _, id := range attachmentIDs { + hash.Write(id[:]) + } + return hash.Sum(nil) +} + func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid.UUID, idempotencyKey string, expectedHash []byte) (model.Message, bool, error) { var message model.Message var createdAt time.Time var storedHash []byte err := s.pool.QueryRow(ctx, ` - SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, i.request_hash + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position), + i.request_hash FROM message_idempotency i JOIN messages m ON m.id = i.message_id JOIN chat_members cm ON cm.chat_id = i.chat_id AND cm.user_id = i.sender_id @@ -102,7 +195,8 @@ func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid AND i.idempotency_key = $3 AND i.expires_at > now()`, senderID, chatID, idempotencyKey).Scan( - &message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt, &storedHash) + &message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt, + &message.AttachmentIDs, &storedHash) if errors.Is(err, pgx.ErrNoRows) { return model.Message{}, false, nil } @@ -112,6 +206,9 @@ func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid if !bytes.Equal(storedHash, expectedHash) { return model.Message{}, false, common.Conflict("idempotency key reused with a different message") } + if message.AttachmentIDs == nil { + message.AttachmentIDs = []uuid.UUID{} + } message.CreatedAt = common.RFCFormTime(createdAt.UTC()) return message, true, nil @@ -136,10 +233,14 @@ func (s *Store) ListMessages(ctx context.Context, userID, chatID uuid.UUID, limi cursorTime, cursorID := cursorValues(cursor) rows, err := s.pool.Query(ctx, ` - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 AND (created_at, id) > ($2, $3) - ORDER BY created_at, id + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position) + FROM messages m + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) + ORDER BY m.created_at, m.id LIMIT $4`, chatID, cursorTime, cursorID, limit+1) if err != nil { return nil, dbError(err) @@ -172,9 +273,13 @@ func (s *Store) ListMessages(ctx context.Context, userID, chatID uuid.UUID, limi func scanMessage(row rowScanner) (model.Message, error) { var message model.Message var createdAt time.Time - if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt); err != nil { + if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, + &createdAt, &message.AttachmentIDs); err != nil { return model.Message{}, err } + if message.AttachmentIDs == nil { + message.AttachmentIDs = []uuid.UUID{} + } message.CreatedAt = common.RFCFormTime(createdAt.UTC()) return message, nil } diff --git a/app/internal/store/messages_test.go b/app/internal/store/messages_test.go new file mode 100644 index 0000000..0363a83 --- /dev/null +++ b/app/internal/store/messages_test.go @@ -0,0 +1,80 @@ +package store + +import ( + "bytes" + "crypto/sha256" + "testing" + + "github.com/google/uuid" +) + +func TestCanonicalAttachmentIDsSortsAndCopies(t *testing.T) { + first, second, third := uuid.MustParse("00000000-0000-0000-0000-000000000003"), + uuid.MustParse("00000000-0000-0000-0000-000000000001"), + uuid.MustParse("00000000-0000-0000-0000-000000000002") + + input := []uuid.UUID{first, second, third} + got := canonicalAttachmentIDs(input) + + want := []uuid.UUID{second, third, first} + if !equalIDs(got, want) { + t.Fatalf("canonicalAttachmentIDs = %v, want %v", got, want) + } + if !equalIDs(input, []uuid.UUID{first, second, third}) { + t.Fatalf("canonicalAttachmentIDs mutated its input: %v", input) + } +} + +func TestCanonicalAttachmentIDsReturnsEmptySliceForNil(t *testing.T) { + got := canonicalAttachmentIDs(nil) + if got == nil { + t.Fatal("canonicalAttachmentIDs(nil) = nil, want empty slice") + } + if len(got) != 0 { + t.Fatalf("canonicalAttachmentIDs(nil) = %v, want empty", got) + } +} + +// TestMessageFingerprintWithoutAttachmentsPreservesLab3Hash is the +// compatibility guard: idempotency rows written before attachments existed +// must still match on replay. +func TestMessageFingerprintWithoutAttachmentsPreservesLab3Hash(t *testing.T) { + expected := sha256.Sum256([]byte("hello")) + if got := messageFingerprint("hello", nil); !bytes.Equal(got, expected[:]) { + t.Fatalf("fingerprint = %x, want %x", got, expected) + } + if got := messageFingerprint("hello", []uuid.UUID{}); !bytes.Equal(got, expected[:]) { + t.Fatalf("empty slice fingerprint = %x, want %x", got, expected) + } +} + +func TestMessageFingerprintDependsOnSortedAttachments(t *testing.T) { + first := uuid.MustParse("00000000-0000-0000-0000-000000000001") + second := uuid.MustParse("00000000-0000-0000-0000-000000000002") + + // CreateMessage canonicalizes before hashing; the pair must therefore be + // order independent even though messageFingerprint itself takes the list + // as given. + one := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{first, second})) + reordered := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{second, first})) + if !bytes.Equal(one, reordered) { + t.Fatal("fingerprint depends on attachment order") + } + + other := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{first})) + if bytes.Equal(one, other) { + t.Fatal("fingerprint ignores attachment identity") + } +} + +func equalIDs(a, b []uuid.UUID) bool { + if len(a) != len(b) { + return false + } + for index := range a { + if a[index] != b[index] { + return false + } + } + return true +} diff --git a/app/internal/users/handler.go b/app/internal/users/handler.go index ebd8675..e30363d 100644 --- a/app/internal/users/handler.go +++ b/app/internal/users/handler.go @@ -11,11 +11,12 @@ import ( ) type Handler struct { - store *store.Store + store *store.Store + jsonBodyLimit int64 } -func NewHandler(st *store.Store) *Handler { - return &Handler{store: st} +func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler { + return &Handler{store: st, jsonBodyLimit: jsonBodyLimit} } type CreateUserRequest struct { @@ -26,7 +27,7 @@ type CreateUserRequest struct { func (h *Handler) CreateUser(c fiber.Ctx) error { var req CreateUserRequest - if err := common.BindJSON(c, &req); err != nil { + if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil { return err } diff --git a/compose.yaml b/compose.yaml index a1bdb6f..8aa9162 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,7 @@ x-api-build: &api-build context: . dockerfile: Dockerfile +# API receives only the attachments/ scoped S3 credential x-api-environment: &api-environment APP_HOST: 0.0.0.0 APP_PORT: "8080" @@ -14,6 +15,15 @@ x-api-environment: &api-environment CACHE_URL: redis://redis:6379/0 SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IMAGE_DECODE_MEMORY_MB: ${IMAGE_DECODE_MEMORY_MB:-256} + UPLOAD_TEMP_DIR: /tmp/messenger-uploads services: postgres: @@ -64,6 +74,67 @@ services: cpus: "${REDIS_CPU_LIMIT:-0.50}" memory: "${REDIS_MEM_LIMIT:-256M}" + rustfs: + image: rustfs/rustfs:1.0.4 + restart: unless-stopped + # AGENT: no published ports. The S3 API is reachable only on the Compose + # network, and the admin console is disabled. For inspection, see the + # one-shot rc client pattern in scripts/check-private-bucket.sh. + environment: + RUSTFS_ACCESS_KEY: ${S3_ROOT_ACCESS_KEY:?set S3_ROOT_ACCESS_KEY in .env} + RUSTFS_SECRET_KEY: ${S3_ROOT_SECRET_KEY:?set S3_ROOT_SECRET_KEY in .env} + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_OBS_LOGGER_LEVEL: "error" + RUSTFS_OBS_LOG_DIRECTORY: "/logs" + command: ["/data"] + volumes: + - rustfsdata:/data + expose: + - "9000" + healthcheck: + test: + [ + "CMD-SHELL", + "wget -q -O - http://127.0.0.1:9000/health/ready >/dev/null", + ] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 15s + deploy: + resources: + limits: + cpus: "${RUSTFS_CPU_LIMIT:-1.0}" + memory: "${RUSTFS_MEM_LIMIT:-512M}" + + # AGENT: bootstrap converges the bucket and the scoped IAM identity on every + # run. Create is idempotent (ignore-existing), policy create replaces the + # named document, and user add updates the password if the user exists. + s3-init: + image: rustfs/rc:v0.1.36 + restart: "no" + depends_on: + rustfs: + condition: service_healthy + environment: + HOME: /tmp + S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ROOT_ACCESS_KEY: ${S3_ROOT_ACCESS_KEY:?set S3_ROOT_ACCESS_KEY in .env} + S3_ROOT_SECRET_KEY: ${S3_ROOT_SECRET_KEY:?set S3_ROOT_SECRET_KEY in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + S3_APP_USER: ${S3_APP_USER:-messenger-api} + S3_APP_POLICY: ${S3_APP_POLICY:-messenger-attachments-rw} + S3_POLICY_TEMPLATE: /config/api-policy.json.template + volumes: + - ./infra/rustfs/bootstrap.sh:/bootstrap.sh:ro + - ./infra/rustfs/api-policy.json.template:/config/api-policy.json.template:ro + entrypoint: ["/bin/sh", "/bootstrap.sh"] + migrate: <<: *api-build restart: "no" @@ -82,6 +153,10 @@ services: condition: service_healthy redis: condition: service_healthy + rustfs: + condition: service_healthy + s3-init: + condition: service_completed_successfully migrate: condition: service_completed_successfully environment: @@ -102,7 +177,35 @@ services: resources: limits: cpus: "${API_CPU_LIMIT:-0.50}" - memory: "${API_MEM_LIMIT:-256M}" + memory: "${API_MEM_LIMIT:-512M}" + + # AGENT: the reconciler uses the same scoped credential as the API. It never + # touches ready attachments; it only removes stale pending rows/objects and + # objects with no metadata row at all. + reconciler: + <<: *api-build + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + rustfs: + condition: service_healthy + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + environment: + <<: *api-environment + RUN_MIGRATIONS: "false" + RECONCILE_ONLY: "true" + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900} + RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-300} + entrypoint: ["/usr/local/bin/reconciler"] + deploy: + resources: + limits: + cpus: "${RECONCILER_CPU_LIMIT:-0.25}" + memory: "${RECONCILER_MEM_LIMIT:-128M}" haproxy: image: haproxy:3.2.6-alpine @@ -152,3 +255,4 @@ services: volumes: pgdata: redisdata: + rustfsdata: diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 0000000..c5a9ffc --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1,4 @@ +{"uploaded_by": "ad8141b5e17c", "attachment_id": "4395a996-db6d-4568-8bfe-805a926b54c8", "size_bytes": 111817} +stopping API replica ad8141b5e17c +{"downloaded_by": "2229551f45ac", "stopped_replica": "ad8141b5e17c", "attempts": 1, "transient_errors": 0, "size_bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"} +PASS: attachment survived the stop of the replica that accepted it diff --git a/evidence/attachments-negative.txt b/evidence/attachments-negative.txt new file mode 100644 index 0000000..48f2a74 --- /dev/null +++ b/evidence/attachments-negative.txt @@ -0,0 +1,2 @@ +python3 tests/custom/attachments.py --base-url "http://haproxy:8080" +PASS: graded negative cases (413/415/404/400), no 5xx, cross-member download checksum matches diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..ef89b76 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://haproxy:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 1, "first_recovery_seconds": 2.557} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..a1c73bf --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,2 @@ +python3 tests/custom/idempotency.py --base-url "http://haproxy:8080" +PASS: 12 concurrent requests used 2 replicas and stored one message diff --git a/evidence/interrupted-upload.txt b/evidence/interrupted-upload.txt new file mode 100644 index 0000000..ba753d3 --- /dev/null +++ b/evidence/interrupted-upload.txt @@ -0,0 +1,4 @@ +attachments rows: 61 -> 61 +stored objects: 61 -> 61 +temp files left: 0 +PASS: interrupted upload left no row, object or temp file diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..306c2f0 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.216, + "concurrency": 8, + "successful_requests": 6249, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 410.689, + "successful_latency_ms": { + "p50": 18.104, + "p95": 31.058, + "p99": 46.769 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..306c2f0 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.216, + "concurrency": 8, + "successful_requests": 6249, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 410.689, + "successful_latency_ms": { + "p50": 18.104, + "p95": 31.058, + "p99": 46.769 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence-containers.txt b/evidence/persistence-containers.txt new file mode 100644 index 0000000..53a3398 --- /dev/null +++ b/evidence/persistence-containers.txt @@ -0,0 +1,17 @@ +before: +/mtusi2026lab-api-1 started=2026-10-31T17:51:06.750192429Z +/mtusi2026lab-api-2 started=2026-10-31T17:51:49.665340634Z +/mtusi2026lab-haproxy-1 started=2026-10-31T17:48:59.281606934Z +/mtusi2026lab-postgres-1 started=2026-10-31T17:48:59.351033349Z +/mtusi2026lab-redis-1 started=2026-10-31T17:48:59.306542581Z +/mtusi2026lab-rustfs-1 started=2026-10-31T17:48:59.424216765Z +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://haproxy:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after: +/mtusi2026lab-api-1 started=2026-10-31T17:52:30.339455795Z +/mtusi2026lab-api-2 started=2026-10-31T17:52:30.289832188Z +/mtusi2026lab-haproxy-1 started=2026-10-31T17:52:30.089676215Z +/mtusi2026lab-postgres-1 started=2026-10-31T17:52:30.233422161Z +/mtusi2026lab-redis-1 started=2026-10-31T17:52:30.195015955Z +/mtusi2026lab-rustfs-1 started=2026-10-31T17:52:29.875749955Z diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..55326c3 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://haproxy:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 0000000..4ea2f4b --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,9 @@ +1/4 create a probe object with the root credential +2/4 anonymous object GET must be denied + anonymous GET -> 403 +3/4 scoped credential works inside attachments/ and is denied elsewhere + scoped PUT/HEAD inside attachments/ -> allowed + scoped PUT outside attachments/ -> denied + scoped CreateBucket -> denied +4/4 cleanup the probe object on exit +PASS: bucket is private and the API credential is limited to attachments/* diff --git a/evidence/readiness-postgres.txt b/evidence/readiness-postgres.txt new file mode 100644 index 0000000..ef09c1f --- /dev/null +++ b/evidence/readiness-postgres.txt @@ -0,0 +1,18 @@ +== before == + X-Instance-Id: 2229551f45ac + Connection: close + +== stopping postgres == + Container mtusi2026lab-postgres-1 Stopping + Container mtusi2026lab-postgres-1 Stopped + HTTP/1.1 503 Service Unavailable +wget: server returned error: HTTP/1.1 503 Service Unavailable +== starting postgres == + Container mtusi2026lab-postgres-1 Starting + Container mtusi2026lab-postgres-1 Started +== after == + X-Request-Id: SixnEMLrvZMdpzQ7KEnqWSq0qLmSnx02kMfwNT39fiA + X-Instance-Id: 2229551f45ac + Connection: close + +{"status":"ok"} \ No newline at end of file diff --git a/evidence/readiness-redis.txt b/evidence/readiness-redis.txt new file mode 100644 index 0000000..2b0b0d8 --- /dev/null +++ b/evidence/readiness-redis.txt @@ -0,0 +1,15 @@ +== before == + Connection: close + +{"status":"ok"}== stopping redis == + Container mtusi2026lab-redis-1 Stopping + Container mtusi2026lab-redis-1 Stopped + HTTP/1.1 503 Service Unavailable +wget: server returned error: HTTP/1.1 503 Service Unavailable +== starting redis == + Container mtusi2026lab-redis-1 Starting + Container mtusi2026lab-redis-1 Started +== after == + X-Instance-Id: 2229551f45ac + Connection: close + diff --git a/evidence/readiness-s3.txt b/evidence/readiness-s3.txt new file mode 100644 index 0000000..fc516bf --- /dev/null +++ b/evidence/readiness-s3.txt @@ -0,0 +1,21 @@ +== before stopping rustfs == + X-Request-Id: JlYsHSjQDyVkf9Sfo3ifA3L0_2k4DXB4ONl34YgKC40 + X-Instance-Id: 2229551f45ac + Connection: close + +{"status":"ok"}exit=0 +== stopping rustfs == + Container mtusi2026lab-rustfs-1 Stopping + Container mtusi2026lab-rustfs-1 Stopped + HTTP/1.1 503 Service Unavailable +wget: server returned error: HTTP/1.1 503 Service Unavailable +exit=0 +== starting rustfs == + Container mtusi2026lab-rustfs-1 Starting + Container mtusi2026lab-rustfs-1 Started +== after rustfs recovery == + X-Request-Id: KeusU6cz8m0fpFPz5NpDzvQU3XD0E6N-bLLo410ePG8 + X-Instance-Id: 2229551f45ac + Connection: close + +{"status":"ok"}exit=0 diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt new file mode 100644 index 0000000..a068f42 --- /dev/null +++ b/evidence/reconcile.txt @@ -0,0 +1,24 @@ + Container mtusi2026lab-reconciler-1 Stopping + Container mtusi2026lab-reconciler-1 Stopped +1/4 old pending row is visible to dry-run but unchanged + dry-run reported it and changed neither store +2/4 old pending row and object are removed by a real pass + Container mtusi2026lab-reconciler-run-d207ba83e1d7 Creating + Container mtusi2026lab-reconciler-run-d207ba83e1d7 Created +2026/11/06 15:57:35 INFO reconcile pass complete dry_run=false stale_pending=1 orphan_objects=0 deleted_rows=1 deleted_objects=1 skipped=0 + row and object removed +3/4 fresh pending row survives the grace period + Container mtusi2026lab-reconciler-run-c71fa380c411 Creating + Container mtusi2026lab-reconciler-run-c71fa380c411 Created +2026/11/06 15:57:39 INFO reconcile pass complete dry_run=false stale_pending=0 orphan_objects=0 deleted_rows=0 deleted_objects=0 skipped=0 + fresh row and object kept + Container mtusi2026lab-reconciler-run-2666c645577b Creating + Container mtusi2026lab-reconciler-run-2666c645577b Created +2026/11/06 15:57:41 INFO reconcile pass complete dry_run=false stale_pending=0 orphan_objects=1 deleted_rows=0 deleted_objects=1 skipped=0 + row removed, orphan object cleaned up +4/4 ready attachment is not touched + Container mtusi2026lab-reconciler-run-5f830168e512 Creating + Container mtusi2026lab-reconciler-run-5f830168e512 Created +2026/11/06 15:57:47 INFO reconcile pass complete dry_run=false stale_pending=0 orphan_objects=0 deleted_rows=0 deleted_objects=0 skipped=0 + ready metadata and bytes kept +PASS: reconciler removed only abandoned data and respected the grace period diff --git a/evidence/s3-bootstrap.txt b/evidence/s3-bootstrap.txt new file mode 100644 index 0000000..7b51170 --- /dev/null +++ b/evidence/s3-bootstrap.txt @@ -0,0 +1,8 @@ +run 1: + Container mtusi2026lab-s3-init-run-6ff27dc7d5c8 Creating + Container mtusi2026lab-s3-init-run-6ff27dc7d5c8 Created +S3 bootstrap complete: bucket=messenger-attachments user=messenger-api policy=messenger-attachments-rw +run 2: + Container mtusi2026lab-s3-init-run-83790bb5cf48 Creating + Container mtusi2026lab-s3-init-run-83790bb5cf48 Created +S3 bootstrap complete: bucket=messenger-attachments user=messenger-api policy=messenger-attachments-rw diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..45d0c03 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://haproxy:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload-memory-run.txt b/evidence/upload-memory-run.txt new file mode 100644 index 0000000..4336622 --- /dev/null +++ b/evidence/upload-memory-run.txt @@ -0,0 +1,25 @@ +{ + "workload": { + "uploads": 10, + "width": 1800, + "height": 1400, + "image_bytes": 7563763, + "sha256": "e42f6283ac02b27d81700f861fc943a1bcb2b17a638edde130d37625315907f5", + "elapsed_seconds": 2.6, + "instances": [ + "2229551f45ac", + "ad8141b5e17c" + ], + "download_verified": true + }, + "sampling_interval_seconds": 0.05, + "samples": 121, + "wall_seconds": 6.884, + "baseline_total_mib": 14.7, + "peak_total_mib": 354.1, + "peak_per_container_mib": { + "mtusi2026lab-api-1": 174.1, + "mtusi2026lab-api-2": 180.1 + }, + "api_memory_limit_mib": 512.0 +} diff --git a/evidence/upload-memory-single-run.txt b/evidence/upload-memory-single-run.txt new file mode 100644 index 0000000..214639d --- /dev/null +++ b/evidence/upload-memory-single-run.txt @@ -0,0 +1,24 @@ +{ + "workload": { + "uploads": 1, + "width": 1800, + "height": 1400, + "image_bytes": 7563763, + "sha256": "be9606c01fdb972e9d60ffdacc03d0208cdca825a0797e727647639b14c0e35c", + "elapsed_seconds": 0.435, + "instances": [ + "2229551f45ac" + ], + "download_verified": true + }, + "sampling_interval_seconds": 0.05, + "samples": 82, + "wall_seconds": 4.616, + "baseline_total_mib": 16.7, + "peak_total_mib": 280.4, + "peak_per_container_mib": { + "mtusi2026lab-api-1": 142.9, + "mtusi2026lab-api-2": 137.5 + }, + "api_memory_limit_mib": 512.0 +} diff --git a/evidence/upload-memory-single.json b/evidence/upload-memory-single.json new file mode 100644 index 0000000..214639d --- /dev/null +++ b/evidence/upload-memory-single.json @@ -0,0 +1,24 @@ +{ + "workload": { + "uploads": 1, + "width": 1800, + "height": 1400, + "image_bytes": 7563763, + "sha256": "be9606c01fdb972e9d60ffdacc03d0208cdca825a0797e727647639b14c0e35c", + "elapsed_seconds": 0.435, + "instances": [ + "2229551f45ac" + ], + "download_verified": true + }, + "sampling_interval_seconds": 0.05, + "samples": 82, + "wall_seconds": 4.616, + "baseline_total_mib": 16.7, + "peak_total_mib": 280.4, + "peak_per_container_mib": { + "mtusi2026lab-api-1": 142.9, + "mtusi2026lab-api-2": 137.5 + }, + "api_memory_limit_mib": 512.0 +} diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..4336622 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,25 @@ +{ + "workload": { + "uploads": 10, + "width": 1800, + "height": 1400, + "image_bytes": 7563763, + "sha256": "e42f6283ac02b27d81700f861fc943a1bcb2b17a638edde130d37625315907f5", + "elapsed_seconds": 2.6, + "instances": [ + "2229551f45ac", + "ad8141b5e17c" + ], + "download_verified": true + }, + "sampling_interval_seconds": 0.05, + "samples": 121, + "wall_seconds": 6.884, + "baseline_total_mib": 14.7, + "peak_total_mib": 354.1, + "peak_per_container_mib": { + "mtusi2026lab-api-1": 174.1, + "mtusi2026lab-api-2": 180.1 + }, + "api_memory_limit_mib": 512.0 +} diff --git a/infra/rustfs/api-policy.json.template b/infra/rustfs/api-policy.json.template new file mode 100644 index 0000000..42c837c --- /dev/null +++ b/infra/rustfs/api-policy.json.template @@ -0,0 +1,26 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::__BUCKET__" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::__BUCKET__/attachments/*" + ] + } + ] +} diff --git a/infra/rustfs/bootstrap.sh b/infra/rustfs/bootstrap.sh new file mode 100644 index 0000000..550bb41 --- /dev/null +++ b/infra/rustfs/bootstrap.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Idempotent RustFS bootstrap: create the private bucket, a scoped IAM user and +# a policy that only allows object operations under the attachments/ prefix. +set -eu + +: "${S3_ENDPOINT:?S3_ENDPOINT is required}" +: "${S3_BUCKET:?S3_BUCKET is required}" +: "${S3_ROOT_ACCESS_KEY:?S3_ROOT_ACCESS_KEY is required}" +: "${S3_ROOT_SECRET_KEY:?S3_ROOT_SECRET_KEY is required}" +: "${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required}" +: "${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required}" +: "${S3_APP_USER:?S3_APP_USER is required}" +: "${S3_APP_POLICY:?S3_APP_POLICY is required}" +: "${S3_POLICY_TEMPLATE:?S3_POLICY_TEMPLATE is required}" + +# AGENT: rc stores its alias file in $HOME. The image runs as an unprivileged +# user, so point HOME at a writable temporary directory. +export HOME="${HOME:-/tmp}" +ALIAS=local + +rc alias set "$ALIAS" "$S3_ENDPOINT" "$S3_ROOT_ACCESS_KEY" "$S3_ROOT_SECRET_KEY" >/dev/null + +# Compose waits for the RustFS healthcheck, but the S3 API can become ready a +# moment later. A bounded wait keeps this service deterministic without hiding +# a genuinely broken deployment. +attempt=0 +until rc ready "$ALIAS" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 60 ]; then + echo "RustFS did not become ready within 60 seconds" >&2 + exit 1 + fi + sleep 1 +done + +rc bucket create -p "$ALIAS/$S3_BUCKET" >/dev/null +# Explicitly converge anonymous access to private. This is both a policy +# assertion and a guard against a bucket created public by a previous run. +rc bucket anonymous set private "$ALIAS/$S3_BUCKET" >/dev/null + +# The policy is rendered from a template because the bucket name is part of +# every resource ARN and is configurable in .env. +sed "s|__BUCKET__|$S3_BUCKET|g" "$S3_POLICY_TEMPLATE" > /tmp/api-policy.json + +# All three admin calls are replaces/updates, not creates that fail on a second +# run: policy create replaces the named document, user add updates the secret, +# and policy attach is idempotent. +rc admin policy create "$ALIAS" "$S3_APP_POLICY" /tmp/api-policy.json >/dev/null +rc admin user add "$ALIAS" "$S3_APP_USER" "$S3_SECRET_ACCESS_KEY" >/dev/null +rc admin policy attach "$ALIAS" "$S3_APP_POLICY" --user "$S3_APP_USER" >/dev/null + +echo "S3 bootstrap complete: bucket=$S3_BUCKET user=$S3_APP_USER policy=$S3_APP_POLICY" diff --git a/scripts/check-private-bucket.sh b/scripts/check-private-bucket.sh new file mode 100755 index 0000000..5c2377b --- /dev/null +++ b/scripts/check-private-bucket.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# Prove the bucket is private outside the API and that the API credential is +# scoped. The probe object is created with the RustFS root credential, fetched +# anonymously (must be 403), accessed with the scoped credential inside the +# allowed prefix, and removed again. +set -eu +cd "$(dirname "$0")/.." + +COMPOSE="docker compose --env-file .env" +TMP_DIR="$PWD/evidence/.tmp-private-bucket.$$" +mkdir -p "$TMP_DIR" +chmod 700 "$TMP_DIR" +TMP_ENV="$TMP_DIR/env" +umask 077 +# AGENT: only the S3 settings go into the temporary env file. The scoped +# credential is intentionally used for most checks; the root pair is needed +# only to create and remove the probe object. +cat > "$TMP_ENV" </dev/null 2>&1 && rc rm -f --quiet "root/$S3_BUCKET/$PROBE"' \ + >/dev/null 2>&1 || true + rm -rf "$TMP_DIR" +} +trap cleanup EXIT INT TERM + +network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)") +[ -n "$network" ] || { + echo "cannot determine the Compose network; is the stack running?" >&2 + exit 2 +} + +# shellcheck disable=SC1090 +. "$TMP_ENV" +PROBE="attachments/_privacy_probe_$$" +URL="http://rustfs:9000/$S3_BUCKET/$PROBE" + +echo "1/4 create a probe object with the root credential" +docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + -v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set root "$S3_ENDPOINT" "$S3_ROOT_ACCESS_KEY" "$S3_ROOT_SECRET_KEY" >/dev/null && + rc cp --quiet /fixtures/sample.png "root/$S3_BUCKET/$PROBE"' + +echo "2/4 anonymous object GET must be denied" +anon_code=$(docker run --rm --network "$network" -e URL="$URL" python:3.12-alpine \ + python3 -c ' +import os, sys, urllib.error, urllib.request +try: + with urllib.request.urlopen(os.environ["URL"], timeout=5) as response: + print(response.status) +except urllib.error.HTTPError as error: + print(error.code) +except Exception as error: + print(type(error).__name__, file=sys.stderr) + print(0) +') +[ "$anon_code" = "403" ] || { + echo "anonymous object GET returned $anon_code, expected 403" >&2 + exit 1 +} +echo " anonymous GET -> 403" + +echo "3/4 scoped credential works inside attachments/ and is denied elsewhere" +docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + -v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$PROBE" && + rc head --quiet "app/$S3_BUCKET/$PROBE" >/dev/null' +echo " scoped PUT/HEAD inside attachments/ -> allowed" + +if docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + -v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/outside-$PROBE"' >/dev/null 2>&1; then + echo "scoped credential wrote outside attachments/, expected denial" >&2 + exit 1 +fi +echo " scoped PUT outside attachments/ -> denied" + +if docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc bucket create -p "app/forbidden-'"$$"'"' >/dev/null 2>&1; then + echo "scoped credential created a bucket, expected denial" >&2 + exit 1 +fi +echo " scoped CreateBucket -> denied" + +echo "4/4 cleanup the probe object on exit" +chmod 700 "$TMP_DIR" 2>/dev/null || true +echo "PASS: bucket is private and the API credential is limited to attachments/*" diff --git a/scripts/measure-upload-memory.py b/scripts/measure-upload-memory.py new file mode 100755 index 0000000..9aff6e5 --- /dev/null +++ b/scripts/measure-upload-memory.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Measure API container memory while concurrent uploads run. + +The HTTP workload runs in a throwaway Python container attached to the Compose +network. This host-side script samples the Docker Engine stats API at a short +interval, because the repository shell cannot always reach published ports and +`docker stats --no-stream` is too slow to catch a short upload burst. +""" +import argparse +import json +import socket +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DOCKER_SOCKET = "/var/run/docker.sock" + + +def docker_json(path): + request = f"GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.connect(DOCKER_SOCKET) + sock.sendall(request.encode()) + data = bytearray() + while True: + part = sock.recv(65536) + if not part: + break + data.extend(part) + head, _, body = bytes(data).partition(b"\r\n\r\n") + if b"transfer-encoding: chunked" not in head.lower(): + return json.loads(body) + + # The Engine API uses chunked transfer encoding for stats and inspect. + decoded = bytearray() + position = 0 + while position < len(body): + line_end = body.find(b"\r\n", position) + if line_end < 0: + break + size = int(body[position:line_end].split(b";", 1)[0], 16) + if size == 0: + break + start = line_end + 2 + decoded.extend(body[start : start + size]) + position = start + size + 2 + return json.loads(bytes(decoded)) + + +def compose(*args): + return subprocess.check_output( + ["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True + ).strip() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--uploads", type=int, default=10) + parser.add_argument("--width", type=int, default=1800) + parser.add_argument("--height", type=int, default=1400) + parser.add_argument("--interval", type=float, default=0.05) + parser.add_argument("--output", type=Path, default=ROOT / "evidence/upload-memory.json") + args = parser.parse_args() + + api_ids = [line for line in compose("ps", "-q", "api").splitlines() if line] + if len(api_ids) < 2: + raise SystemExit("at least two running API replicas are required") + names = { + container_id: docker_json(f"/containers/{container_id}/json")["Name"].lstrip("/") + for container_id in api_ids + } + network = docker_json( + f"/containers/{compose('ps', '-q', 'haproxy')}/json" + )["NetworkSettings"]["Networks"] + network_name = next(iter(network)) + + stop = threading.Event() + samples = [] + + def sample(): + while not stop.is_set(): + per_container = {} + for container_id in api_ids: + stats = docker_json( + f"/containers/{container_id}/stats?stream=false&one-shot=true" + ) + per_container[names[container_id]] = stats.get("memory_stats", {}).get( + "usage", 0 + ) + samples.append( + { + "at": time.time(), + "total": sum(per_container.values()), + "per_container": per_container, + } + ) + time.sleep(args.interval) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + time.sleep(0.3) # capture an idle baseline before the burst + + command = [ + "docker", + "run", + "--rm", + "--network", + network_name, + "-v", + f"{ROOT}:/work:ro", + "-w", + "/work", + "python:3.12-alpine", + "python3", + "tests/custom/concurrent_uploads.py", + "--base-url", + "http://haproxy:8080", + "--uploads", + str(args.uploads), + "--width", + str(args.width), + "--height", + str(args.height), + ] + started = time.monotonic() + try: + output = subprocess.check_output(command, text=True) + finally: + stop.set() + sampler.join(timeout=2) + + workload = json.loads(output.strip().splitlines()[-1]) + baseline = samples[0]["total"] if samples else 0 + peak = max(samples, key=lambda item: item["total"]) if samples else {"total": 0} + report = { + "workload": workload, + "sampling_interval_seconds": args.interval, + "samples": len(samples), + "wall_seconds": round(time.monotonic() - started, 3), + "baseline_total_mib": round(baseline / (1024 * 1024), 1), + "peak_total_mib": round(peak["total"] / (1024 * 1024), 1), + "peak_per_container_mib": { + name: round(value / (1024 * 1024), 1) + for name, value in peak["per_container"].items() + }, + "api_memory_limit_mib": None, + } + # The limit is configured in .env; keep the evidence honest by reading the + # effective value from Compose rather than repeating a default. + configured = subprocess.check_output( + [ + "docker", + "inspect", + "-f", + "{{.HostConfig.Memory}}", + api_ids[0], + ], + text=True, + ).strip() + limit_bytes = int(configured or 0) + report["api_memory_limit_mib"] = ( + round(limit_bytes / (1024 * 1024), 1) if limit_bytes else None + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh index 0331aa5..38c4bc1 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,4 +1,4 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose --env-file .env restart postgres redis haproxy api +docker compose --env-file .env restart postgres redis rustfs haproxy api diff --git a/scripts/verify-attachment-failover.sh b/scripts/verify-attachment-failover.sh new file mode 100755 index 0000000..f02e2c6 --- /dev/null +++ b/scripts/verify-attachment-failover.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Verify the lab4 durability claim: upload through the load balancer, stop the +# API replica that served the upload, then download the same attachment through +# another replica and compare the SHA-256. The stopped replica is restored on +# exit so the stand is left as it was found. +set -eu +cd "$(dirname "$0")/.." + +COMPOSE="docker compose --env-file .env" +STATE_DIR="$PWD/evidence/.tmp-attachment-failover.$$" +mkdir -p "$STATE_DIR" +chmod 700 "$STATE_DIR" + +restore() { + rm -rf "$STATE_DIR" + $COMPOSE up -d --scale api=2 api >/dev/null 2>&1 || true + attempts=0 + while [ "$attempts" -lt 60 ]; do + ids=$($COMPOSE ps -q api) + count=$(printf '%s\n' "$ids" | sed '/^$/d' | wc -l) + healthy=$(for id in $ids; do docker inspect -f '{{.State.Health.Status}}' "$id"; done | grep -c '^healthy$' || true) + if [ "$count" -eq 2 ] && [ "$healthy" -eq 2 ]; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 1 + done + echo "warning: API replicas did not return to two healthy instances" >&2 +} +trap restore EXIT INT TERM + +network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)") +[ -n "$network" ] || { + echo "cannot determine the Compose network; is the stack running?" >&2 + exit 2 +} + +run_python() { + docker run --rm --network "$network" \ + -v "$PWD:/work:ro" -v "$STATE_DIR:/state" -w /work \ + python:3.12-alpine \ + python3 tests/custom/attachment_flow.py --base-url http://haproxy:8080 "$@" +} + +upload_json=$(run_python --action upload --state /state/upload.json) +printf '%s\n' "$upload_json" +instance=$(printf '%s' "$upload_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["uploaded_by"])') + +echo "stopping API replica $instance" +TARGET_INSTANCE_ID="$instance" scripts/stop-one.sh + +run_python --action download --state /state/upload.json --timeout 30 + +echo "PASS: attachment survived the stop of the replica that accepted it" diff --git a/scripts/verify-interrupted-upload.sh b/scripts/verify-interrupted-upload.sh new file mode 100755 index 0000000..4ecb737 --- /dev/null +++ b/scripts/verify-interrupted-upload.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# A client that declares a large body and disconnects halfway must not leave a +# pending metadata row, an S3 object or a temp file behind. The handler creates +# the pending row only after the whole bounded body has been copied, so this is +# the expected clean failure. +set -eu +cd "$(dirname "$0")/.." + +COMPOSE="docker compose --env-file .env" +network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)") +[ -n "$network" ] || { + echo "cannot determine the Compose network; is the stack running?" >&2 + exit 2 +} + +POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env) +POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env) +POSTGRES_USER=${POSTGRES_USER:-messenger} +POSTGRES_DB=${POSTGRES_DB:-messenger} + +count_rows() { + $COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc \ + "SELECT count(*) FROM attachments" | tr -d '\r' +} + +count_objects() { + docker run --rm --network "$network" --env-file .env -e HOME=/tmp \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc bucket list -r --no-color "app/$S3_BUCKET/attachments/" | wc -l' | tr -d '\r ' +} + +rows_before=$(count_rows) +objects_before=$(count_objects) + +# The helper prints the chat id and token for this one short-lived probe. The +# token is passed through the environment, not written to disk or evidence. +state=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \ + python:3.12-alpine python3 -c ' +import sys +sys.path.insert(0, "tests") +from client import Client +alice, _, _, chat = Client("http://haproxy:8080").bootstrap() +print(chat["id"] + " " + alice["token"]) +') +chat_id=$(printf '%s' "$state" | cut -d' ' -f1) +token=$(printf '%s' "$state" | cut -d' ' -f2) + +docker run --rm --network "$network" -e CHAT_ID="$chat_id" -e TOKEN="$token" \ + python:3.12-alpine python3 -c ' +import os, socket, time +host, port = "haproxy", 8080 +request = ( + "POST /api/v1/chats/" + os.environ["CHAT_ID"] + "/attachments?filename=cut.png HTTP/1.1\r\n" + "Host: haproxy\r\n" + "Authorization: Bearer " + os.environ["TOKEN"] + "\r\n" + "Content-Type: image/png\r\n" + "Content-Length: 5000000\r\n" + "Connection: close\r\n\r\n" +).encode() +with socket.create_connection((host, port), timeout=5) as sock: + sock.sendall(request) + sock.sendall(b"\x89PNG\r\n\x1a\n" + b"\x00" * 1024) + time.sleep(0.2) +# close without sending the declared body +' +sleep 2 + +rows_after=$(count_rows) +objects_after=$(count_objects) +temp_files=0 +for id in $($COMPOSE ps -q api); do + count=$(docker exec "$id" sh -c 'find /tmp/messenger-uploads -type f 2>/dev/null | wc -l' | tr -d '\r ') + temp_files=$((temp_files + count)) +done + +echo "attachments rows: $rows_before -> $rows_after" +echo "stored objects: $objects_before -> $objects_after" +echo "temp files left: $temp_files" +[ "$rows_before" = "$rows_after" ] || { + echo "interrupted upload left a metadata row" >&2 + exit 1 +} +[ "$objects_before" = "$objects_after" ] || { + echo "interrupted upload left an S3 object" >&2 + exit 1 +} +[ "$temp_files" = "0" ] || { + echo "interrupted upload left a temporary file" >&2 + exit 1 +} +echo "PASS: interrupted upload left no row, object or temp file" diff --git a/scripts/verify-reconcile.sh b/scripts/verify-reconcile.sh new file mode 100755 index 0000000..e402e59 --- /dev/null +++ b/scripts/verify-reconcile.sh @@ -0,0 +1,199 @@ +#!/bin/sh +# Exercise the two-store reconciler without breaking real data. +# +# 1. An old pending row plus its object must be removed; a dry run must not. +# 2. A fresh pending row must survive the grace period. +# 3. An object with no metadata row (orphan) must be removed. +# 4. A ready attachment referenced by PostgreSQL must never be removed. +set -eu +cd "$(dirname "$0")/.." + +COMPOSE="docker compose --env-file .env" +TMP_DIR="$PWD/evidence/.tmp-reconcile.$$" +mkdir -p "$TMP_DIR" +chmod 700 "$TMP_DIR" +TMP_ENV="$TMP_DIR/env" +umask 077 +cat > "$TMP_ENV" <&2 + exit 2 +} +# shellcheck disable=SC1090 +. "$TMP_ENV" + +POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env) +POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env) +POSTGRES_USER=${POSTGRES_USER:-messenger} +POSTGRES_DB=${POSTGRES_DB:-messenger} + +network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)") +[ -n "$network" ] || { + echo "cannot determine the Compose network; is the stack running?" >&2 + exit 2 +} + +# The long-running reconciler has a 900 s grace period. Pause it so the probe +# rows cannot be removed by a background pass while they are being asserted. +$COMPOSE stop reconciler >/dev/null + +psql_value() { + $COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$1" | tr -d '\r' +} + +rc_shell() { + docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && '"$1" +} + +object_exists() { + rc_shell "rc head --quiet \"app/\$S3_BUCKET/$1\" >/dev/null" >/dev/null 2>&1 +} + +put_object() { + docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ + -v "$PWD/tests/fixtures:/fixtures:ro" -e KEY="$1" \ + --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ + 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && + rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$KEY"' +} + +remove_object() { + rc_shell "rc rm -f --quiet \"app/\$S3_BUCKET/$1\"" >/dev/null 2>&1 || true +} + +insert_pending() { + id=$1 + key=$2 + age_sql=$3 + psql_value "INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, created_at, updated_at) + VALUES ('$id', '$CHAT_ID', '$OWNER_ID', 'probe.png', 'image/png', + $(wc -c < tests/fixtures/sample.png), decode('$(sha256sum tests/fixtures/sample.png | cut -d' ' -f1)', 'hex'), + '$key', 'pending', now() - $age_sql, now() - $age_sql)" >/dev/null +} + +run_reconciler() { + $COMPOSE run --rm --no-deps -T reconciler --once "$@" +} + +cleanup() { + for id in ${STALE_ID:-} ${FRESH_ID:-} ${READY_ID:-}; do + [ -n "$id" ] || continue + psql_value "DELETE FROM attachments WHERE id = '$id'" >/dev/null 2>&1 || true + done + for key in ${STALE_KEY:-} ${FRESH_KEY:-} ${READY_KEY:-}; do + [ -n "$key" ] || continue + remove_object "$key" + done + $COMPOSE start reconciler >/dev/null 2>&1 || true + rm -rf "$TMP_DIR" +} +trap cleanup EXIT INT TERM + +# A baseline chat/user pair makes the inserted rows valid without depending on +# data left by an earlier test run. +ids=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \ + python:3.12-alpine python3 -c ' +import sys +sys.path.insert(0, "tests") +from client import Client +alice, _, _, chat = Client("http://haproxy:8080").bootstrap() +print(chat["id"], alice["id"]) +') +CHAT_ID=$(printf '%s' "$ids" | cut -d' ' -f1) +OWNER_ID=$(printf '%s' "$ids" | cut -d' ' -f2) + +echo "1/4 old pending row is visible to dry-run but unchanged" +STALE_ID=$(python3 -c 'import uuid; print(uuid.uuid4())') +STALE_KEY="attachments/$CHAT_ID/$STALE_ID" +put_object "$STALE_KEY" +insert_pending "$STALE_ID" "$STALE_KEY" "interval '1 hour'" +dry=$(run_reconciler --dry-run --grace=1s 2>&1) +printf '%s\n' "$dry" | grep -q "would remove stale pending attachment" || { + echo "dry run did not report the stale pending attachment" >&2 + exit 1 +} +[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "1" ] || { + echo "dry run deleted the pending row" >&2 + exit 1 +} +object_exists "$STALE_KEY" || { + echo "dry run deleted the pending object" >&2 + exit 1 +} +echo " dry-run reported it and changed neither store" + +echo "2/4 old pending row and object are removed by a real pass" +run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/stale.log" +[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "0" ] || { + echo "stale pending row survived" >&2 + exit 1 +} +if object_exists "$STALE_KEY"; then + echo "stale pending object survived" >&2 + exit 1 +fi +echo " row and object removed" + +echo "3/4 fresh pending row survives the grace period" +FRESH_ID=$(python3 -c 'import uuid; print(uuid.uuid4())') +FRESH_KEY="attachments/$CHAT_ID/$FRESH_ID" +put_object "$FRESH_KEY" +insert_pending "$FRESH_ID" "$FRESH_KEY" "interval '0 seconds'" +run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/fresh.log" +[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$FRESH_ID'")" = "1" ] || { + echo "fresh pending row was removed" >&2 + exit 1 +} +object_exists "$FRESH_KEY" || { + echo "fresh pending object was removed" >&2 + exit 1 +} +echo " fresh row and object kept" + +# Remove the row only; the object now has no metadata row and becomes an +# orphan. A very small grace period makes the old object eligible immediately. +psql_value "DELETE FROM attachments WHERE id = '$FRESH_ID'" >/dev/null +run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/orphan.log" +if object_exists "$FRESH_KEY"; then + echo "orphan object survived" >&2 + exit 1 +fi +echo " row removed, orphan object cleaned up" + +echo "4/4 ready attachment is not touched" +READY_ID=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \ + python:3.12-alpine python3 -c ' +import sys +sys.path.insert(0, "tests") +from client import Client +c = Client("http://haproxy:8080") +alice, _, _, chat = c.bootstrap() +att, _ = c.attachment(alice, chat) +print(att["id"]) +') +READY_KEY=$(psql_value "SELECT object_key FROM attachments WHERE id = '$READY_ID'") +[ -n "$READY_KEY" ] || { + echo "could not read the ready object key" >&2 + exit 1 +} +run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/ready.log" +[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$READY_ID'")" = "1" ] || { + echo "ready attachment row was removed" >&2 + exit 1 +} +object_exists "$READY_KEY" || { + echo "ready attachment object was removed" >&2 + exit 1 +} +echo " ready metadata and bytes kept" + +echo "PASS: reconciler removed only abandoned data and respected the grace period" diff --git a/tests/custom/attachment_flow.py b/tests/custom/attachment_flow.py new file mode 100755 index 0000000..77866fe --- /dev/null +++ b/tests/custom/attachment_flow.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Two-step attachment durability probe used by verify-attachment-failover.sh. + +Step 1 (`upload`) uploads a fixture and writes a short-lived state file. The +caller then stops the replica named in `instance_id`. Step 2 (`download`) uses +the same session token through the load balancer, retries transient errors and +checks that another replica returns the exact original bytes. + +The state file contains a test session token. It lives in a private temporary +directory and is never copied into evidence. +""" +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check # noqa: E402 + + +def upload(args): + c = Client(args.base_url, args.ca_file) + alice, _, _, chat = c.bootstrap() + content = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + response = c.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 201, + actor=alice, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + attachment = response["json"] + check(attachment["status"] == "ready", "upload is not ready") + check( + attachment["sha256"] == hashlib.sha256(content).hexdigest(), + "upload checksum mismatch", + ) + state = { + "attachment_id": attachment["id"], + "sha256": attachment["sha256"], + "size_bytes": attachment["size_bytes"], + "instance_id": response["headers"].get("X-Instance-Id"), + "token": alice["token"], + "chat_id": chat["id"], + "content_length": len(content), + } + check(bool(state["instance_id"]), "upload response has no instance id") + args.state.write_text(json.dumps(state)) + print( + json.dumps( + { + "uploaded_by": state["instance_id"], + "attachment_id": state["attachment_id"], + "size_bytes": state["size_bytes"], + } + ) + ) + + +def download(args): + state = json.loads(args.state.read_text()) + expected_sha = state["sha256"] + expected_size = state["size_bytes"] + actor = {"token": state["token"]} + c = Client(args.base_url, args.ca_file) + + deadline = time.monotonic() + args.timeout + attempts = 0 + errors = [] + while time.monotonic() < deadline: + attempts += 1 + try: + response = c.call( + "GET", + f"/api/v1/attachments/{state['attachment_id']}/content", + actor=actor, + ) + except Failure as error: + errors.append(str(error)) + time.sleep(0.5) + continue + + served_by = response["headers"].get("X-Instance-Id") + check(served_by is not None, "download response has no instance id") + check(served_by != state["instance_id"], "download was served by the stopped replica") + check( + len(response["bytes"]) == expected_size, + f"download size {len(response['bytes'])} != {expected_size}", + ) + check( + hashlib.sha256(response["bytes"]).hexdigest() == expected_sha, + "download checksum differs from the uploaded original", + ) + print( + json.dumps( + { + "downloaded_by": served_by, + "stopped_replica": state["instance_id"], + "attempts": attempts, + "transient_errors": len(errors), + "size_bytes": len(response["bytes"]), + "sha256": expected_sha, + } + ) + ) + return + + raise Failure( + f"download did not recover within {args.timeout}s after {attempts} attempts: " + + "; ".join(errors[-3:]) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--action", choices=("upload", "download"), required=True) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--ca-file", default=None) + parser.add_argument("--timeout", type=float, default=30) + args = parser.parse_args() + + if args.action == "upload": + upload(args) + else: + download(args) + + +if __name__ == "__main__": + main() diff --git a/tests/custom/attachments.py b/tests/custom/attachments.py new file mode 100755 index 0000000..922d39f --- /dev/null +++ b/tests/custom/attachments.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Grade-4 negative checks for the attachment path. + +The public smoke covers the happy path. This test adds the failure cases the +assignment requires: an empty body, an oversized body, a corrupted image, a +MIME spoof, a foreign attachment id and an attachment that belongs to another +chat. Every client mistake must remain a 4xx response with the contract error +envelope, not a 500. +""" +import argparse +import hashlib +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, check # noqa: E402 + +FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 + + +def upload(client, actor, chat_id, raw, filename, content_type): + headers = {} if content_type is None else {"Content-Type": content_type} + return client.call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + expected=(201, 400, 404, 413, 415), + actor=actor, + raw=raw, + headers=headers, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--ca-file", default=None) + args = parser.parse_args() + + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + png = FIXTURE.read_bytes() + + # Happy path first, so the negative checks can refer to a real attachment. + good = c.attachment(alice, chat)[0] + check(good["status"] == "ready", "valid upload is not ready") + check(good["error_code"] is None, "ready attachment carries an error") + + # 1. One byte over the contract limit must be a 413, decided before decode. + oversized = b"\x00" * (MAX_UPLOAD_BYTES + 1) + response = upload(c, alice, chat["id"], oversized, "big.png", "image/png") + check(response["status"] == 413, "oversized upload did not return 413") + + # 2. A file with a valid PNG signature but a broken body must be a 415. + corrupt = png[:8] + b"\x00" * 64 + response = upload(c, alice, chat["id"], corrupt, "broken.png", "image/png") + check(response["status"] == 415, "corrupted image did not return 415") + + # 3. The declared type disagrees with the actual PNG bytes: MIME spoof. + response = upload(c, alice, chat["id"], png, "spoof.jpg", "image/jpeg") + check(response["status"] == 415, "MIME spoof did not return 415") + + # 4. A non-image content type is rejected without touching the decoder. + response = upload(c, alice, chat["id"], b"hello", "file.txt", "text/plain") + check(response["status"] == 415, "non-image Content-Type did not return 415") + + # 4b. An empty body is rejected before validation and leaves no metadata. + response = upload(c, alice, chat["id"], b"", "empty.png", "image/png") + check(response["status"] == 415, "empty upload did not return 415") + + # 5. Path separators in the display name are rejected before I/O. + for bad_name in ("../escape.png", "dir/photo.png", "back\\slash.png"): + response = upload(c, alice, chat["id"], png, bad_name, "image/png") + check(response["status"] == 400, f"filename {bad_name!r} did not return 400") + + # 6. Eve is not a chat member: both metadata and bytes are hidden as 404. + for suffix in ("", "/content"): + response = c.call( + "GET", f"/api/v1/attachments/{good['id']}{suffix}", expected=404, actor=eve + ) + check( + response["json"]["error"]["code"] == "not_found", + "foreign attachment error code mismatch", + ) + + # 7. An attachment of another chat cannot be linked to this chat. + other = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + other_attachment = c.attachment(alice, other)[0] + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 404, + actor=alice, + body={"text": "wrong chat", "attachment_ids": [other_attachment["id"]]}, + headers={"Idempotency-Key": "wrong-chat-" + uuid.uuid4().hex}, + ) + c.call( + "POST", + f"/api/v1/chats/{other['id']}/messages", + 404, + actor=alice, + body={"text": "wrong chat", "attachment_ids": [good["id"]]}, + headers={"Idempotency-Key": "wrong-chat-" + uuid.uuid4().hex}, + ) + + # 8. Unknown ids, duplicates and an oversized list are cheap 400/404s. + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 404, + actor=alice, + body={"text": "unknown", "attachment_ids": [str(uuid.uuid4())]}, + headers={"Idempotency-Key": "unknown-" + uuid.uuid4().hex}, + ) + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + body={"text": "duplicate", "attachment_ids": [good["id"], good["id"]]}, + headers={"Idempotency-Key": "duplicate-" + uuid.uuid4().hex}, + ) + c.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + body={"text": "many", "attachment_ids": [str(uuid.uuid4())] * 11}, + headers={"Idempotency-Key": "many-" + uuid.uuid4().hex}, + ) + + # 9. Uploading into a chat the caller does not belong to is a 404, and the + # response body must not become a 500 even though authorization wins + # before the body is read. + response = upload(c, eve, chat["id"], png, "foreign.png", "image/png") + check(response["status"] == 404, "foreign-chat upload did not return 404") + + # 10. A valid image attached to a message is visible to the other member + # with the exact original checksum, independently of the uploader. + message = c.message(alice, chat, "", attachments=[good["id"]])["json"] + check(message["attachment_ids"] == [good["id"]], "message lost attachment id") + download = c.call( + "GET", f"/api/v1/attachments/{good['id']}/content", actor=bob + ) + check(download["bytes"] == png, "member download differs from original") + check( + hashlib.sha256(download["bytes"]).hexdigest() == good["sha256"], + "member download checksum mismatch", + ) + + print( + "PASS: graded negative cases (413/415/404/400), no 5xx, " + "cross-member download checksum matches" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/custom/concurrent_uploads.py b/tests/custom/concurrent_uploads.py new file mode 100644 index 0000000..59e0da6 --- /dev/null +++ b/tests/custom/concurrent_uploads.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Upload ten or more large-but-valid PNGs at the same time. + +The payload is generated in memory and shared read-only between threads, so the +client itself does not multiply the image. This keeps the measurement focused on +API memory: streaming temp files plus bounded image decoding. +""" +import argparse +import hashlib +import json +import os +import struct +import sys +import threading +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, check # noqa: E402 + + +def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + +def make_png(width, height): + compressor = zlib.compressobj(1) + rows = bytearray() + for _ in range(height): + rows += b"\x00" + os.urandom(width * 3) + body = compressor.compress(bytes(rows)) + compressor.flush() + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", body) + + chunk(b"IEND", b"") + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--ca-file", default=None) + parser.add_argument("--uploads", type=int, default=10) + parser.add_argument("--width", type=int, default=1800) + parser.add_argument("--height", type=int, default=1400) + args = parser.parse_args() + + c = Client(args.base_url, args.ca_file) + alice, _, _, chat = c.bootstrap() + png = make_png(args.width, args.height) + digest = hashlib.sha256(png).hexdigest() + check(len(png) <= 10 * 1024 * 1024, "generated fixture exceeds the upload limit") + + barrier = threading.Barrier(args.uploads) + results = [] + errors = [] + lock = threading.Lock() + + def worker(index): + client = Client(args.base_url, args.ca_file) + try: + barrier.wait(timeout=30) + response = client.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=concurrent-{index}.png", + 201, + actor=alice, + raw=png, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + with lock: + results.append( + { + "id": response["json"]["id"], + "instance": response["headers"].get("X-Instance-Id"), + } + ) + except Exception as error: # noqa: BLE001 - report every worker failure + with lock: + errors.append(f"{type(error).__name__}: {error}") + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.uploads)] + started = time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join() + elapsed = time.monotonic() - started + + check(not errors, "concurrent uploads failed: " + "; ".join(errors[:3])) + check(len(results) == args.uploads, "not all concurrent uploads completed") + instances = sorted({item["instance"] for item in results if item["instance"]}) + check(bool(instances), "no X-Instance-Id was observed") + + # Read one uploaded object back and compare the exact bytes before reporting + # success; ten 201 responses alone are not enough. + download = c.call( + "GET", f"/api/v1/attachments/{results[0]['id']}/content", actor=alice + ) + check( + hashlib.sha256(download["bytes"]).hexdigest() == digest, + "downloaded object differs from the generated original", + ) + + report = { + "uploads": args.uploads, + "width": args.width, + "height": args.height, + "image_bytes": len(png), + "sha256": digest, + "elapsed_seconds": round(elapsed, 3), + "instances": instances, + "download_verified": True, + } + print(json.dumps(report)) + + +if __name__ == "__main__": + main()