From f9444a396568c9115c945e17914dda60d1cefec6 Mon Sep 17 00:00:00 2001 From: d1frend Date: Fri, 30 Oct 2026 12:27:49 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BB=D0=B0=D0=B1=D0=B0=204=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D0=BE=D1=86=D0=B5=D0=BD=D0=BA=D1=83=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 8 +- Dockerfile | 4 +- Makefile | 30 +- REPORT.md | 120 +++- cmd/reconciler/main.go | 75 +++ cmd/server/main.go | 37 +- compose.yaml | 96 +++- evidence/lab4-results.md | 19 + frontend/src/api/client.ts | 9 +- frontend/src/api/types.ts | 1 + go.mod | 14 + go.sum | 34 ++ infrastructure/haproxy/haproxy.cfg | 4 +- infrastructure/minio/Dockerfile | 40 ++ infrastructure/minio/api-policy.template.json | 26 + internal/adapter/health/multi.go | 33 ++ internal/adapter/memory/repository.go | 84 ++- internal/adapter/memory/store.go | 96 +++- internal/adapter/objectstore/minio.go | 201 +++++++ internal/adapter/objectstore/minio_test.go | 37 ++ internal/adapter/postgres/attachments.go | 306 ++++++++++ internal/adapter/postgres/messages.go | 116 +++- internal/application/attachment_queries.go | 65 +++ internal/application/attachments.go | 246 ++++++++ internal/application/attachments_test.go | 524 ++++++++++++++++++ internal/application/commands.go | 11 + internal/application/cursor_test.go | 34 +- internal/application/errors.go | 15 +- internal/application/messages.go | 15 +- internal/application/ports.go | 28 + internal/application/service.go | 40 +- internal/config/config.go | 31 ++ internal/domain/attachment.go | 119 ++++ internal/domain/attachment_test.go | 59 ++ internal/domain/message.go | 11 +- internal/domain/message_test.go | 70 +++ internal/domain/validation.go | 34 +- internal/reconcile/reconciler.go | 229 ++++++++ internal/reconcile/reconciler_test.go | 209 +++++++ internal/transport/http/dto/attachment.go | 59 ++ internal/transport/http/dto/message.go | 21 +- .../transport/http/handlers/create_message.go | 6 + .../transport/http/handlers/get_attachment.go | 94 ++++ .../http/handlers/upload_attachment.go | 97 ++++ migrations/000003_attachments.down.sql | 36 ++ migrations/000003_attachments.up.sql | 191 +++++++ scripts/init-secrets.sh | 39 ++ scripts/restart.sh | 17 +- scripts/stop-one.sh | 10 +- student_tests/application_helper_test.go | 8 +- student_tests/attachment_upload_test.go | 256 +++++++++ student_tests/idempotency_test.go | 42 +- student_tests/message_attachments_test.go | 133 +++++ tests/attachment_boundaries.py | 99 ++++ tests/attachment_memory.py | 149 +++++ tests/minio_policy.sh | 52 ++ tests/reconcile_integration.py | 176 ++++++ tests/reconcile_scale.py | 90 +++ tests/storage_outage.py | 81 +++ 59 files changed, 4696 insertions(+), 90 deletions(-) create mode 100644 cmd/reconciler/main.go create mode 100644 evidence/lab4-results.md create mode 100644 infrastructure/minio/Dockerfile create mode 100644 infrastructure/minio/api-policy.template.json create mode 100644 internal/adapter/health/multi.go create mode 100644 internal/adapter/objectstore/minio.go create mode 100644 internal/adapter/objectstore/minio_test.go create mode 100644 internal/adapter/postgres/attachments.go create mode 100644 internal/application/attachment_queries.go create mode 100644 internal/application/attachments.go create mode 100644 internal/application/attachments_test.go create mode 100644 internal/domain/attachment.go create mode 100644 internal/domain/attachment_test.go create mode 100644 internal/domain/message_test.go create mode 100644 internal/reconcile/reconciler.go create mode 100644 internal/reconcile/reconciler_test.go create mode 100644 internal/transport/http/dto/attachment.go create mode 100644 internal/transport/http/handlers/get_attachment.go create mode 100644 internal/transport/http/handlers/upload_attachment.go create mode 100644 migrations/000003_attachments.down.sql create mode 100644 migrations/000003_attachments.up.sql create mode 100755 scripts/init-secrets.sh create mode 100644 student_tests/attachment_upload_test.go create mode 100644 student_tests/message_attachments_test.go create mode 100644 tests/attachment_boundaries.py create mode 100644 tests/attachment_memory.py create mode 100755 tests/minio_policy.sh create mode 100644 tests/reconcile_integration.py create mode 100644 tests/reconcile_scale.py create mode 100644 tests/storage_outage.py diff --git a/.env.example b/.env.example index d99d26a..dd022e2 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,8 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. # These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 -APP_PORT=8080 +APP_INTERNAL_PORT=8080 +PUBLIC_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. @@ -11,10 +12,13 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_OPERATION_TIMEOUT=5s MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Keep the Go soft heap limit below the Compose hard limit (256 MiB by default). +API_GO_MEMORY_LIMIT=220MiB diff --git a/Dockerfile b/Dockerfile index a62b7bc..987bb77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ COPY internal/ ./internal/ RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/server ./cmd/server/main.go +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/reconciler ./cmd/reconciler/main.go FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 @@ -19,6 +20,7 @@ RUN addgroup -g 1000 appgroup && \ WORKDIR /app COPY --from=builder /build/server /app/server +COPY --from=builder /build/reconciler /app/reconciler RUN chown -R appuser:appgroup /app @@ -30,4 +32,4 @@ HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=10 \ CMD wget -qO- http://127.0.0.1:8080/health/ready || exit 1 -ENTRYPOINT ["/app/server"] \ No newline at end of file +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile index 5d86d17..c2149e7 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,17 @@ PYTHON ?= python3 BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= +RECONCILE_GRACE ?= 15m TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help init-secrets check validate test load test-attachment-boundaries test-attachment-memory test-storage-outage test-minio-policy test-persistence test-failover test-reconcile test-reconcile-scale reconcile-dry-run reconcile-apply help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" +init-secrets: + ./scripts/init-secrets.sh + check: $(PYTHON) scripts/check_contract.py @@ -21,6 +25,18 @@ test: load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachment-memory: + BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_memory.py + +test-attachment-boundaries: + BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_boundaries.py + +test-storage-outage: + BASE_URL="$(BASE_URL)" $(PYTHON) tests/storage_outage.py + +test-minio-policy: + ./tests/minio_policy.sh + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +44,15 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + $(PYTHON) tests/reconcile_integration.py + +test-reconcile-scale: + $(PYTHON) tests/reconcile_scale.py + +reconcile-dry-run: + docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)" + +reconcile-apply: + docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)" --apply diff --git a/REPORT.md b/REPORT.md index 9ad2b84..160bf45 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,123 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: TODO — заполнить перед сдачей. +- Целевая оценка: **5**, выбран **трек 2 — согласованность PostgreSQL и S3**. +- Commit/PR: TODO после публикации. +- Использованные источники и библиотеки: Go standard library, `pgx/v5`, `minio-go/v7`, официальный исходный код MinIO и `mc`, Redis Go client. OpenAI Codex использовался для анализа контракта, проектирования, реализации и тестов; все утверждения ниже проверены перечисленными воспроизводимыми командами. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker с Compose v2, GNU Make, Python 3.9+, OpenSSL. Локально проверено в WSL2. MinIO и `mc` собираются из официальных Go-модулей в [infrastructure/minio/Dockerfile](infrastructure/minio/Dockerfile): сервер `RELEASE.2025-10-15T17-29-55Z`, клиент `RELEASE.2025-08-13T08-35-41Z`. Базовые образы закреплены digest. + +```sh +cp .env.example .env +make init-secrets +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +docker compose --env-file .env ps +BASE_URL=http://127.0.0.1:8080 make test +python3 -m venv .venv +.venv/bin/pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +`make init-secrets` создаёт недостающие файлы с mode `600` и никогда не перезаписывает существующие значения. Секреты находятся в игнорируемом каталоге `secrets/` и не передаются через переменные окружения контейнера. Если `8080` занят, задаётся `PUBLIC_PORT`, например `PUBLIC_PORT=18080`; внутренний `APP_INTERNAL_PORT=8080` от него не зависит. + +Безопасная остановка сохраняет named volumes: + +```sh +docker compose --env-file .env down +``` + +Намеренное удаление всех учебных данных выполняется только явно: + +```sh +docker compose --env-file .env down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Путь запроса: клиент → HAProxy → две API-реплики → PostgreSQL (источник истины метаданных/сессий) + приватный MinIO bucket (байты); Redis остаётся восстанавливаемым кешем чатов. У каждой API-реплики нет постоянного каталога оригиналов. Во время синхронного upload тело ограниченно копируется в файл OS temp, файл имеет случайное имя, удаляется в `defer` и нужен только для двукратной проверки/потокового S3 PUT. + +Из lab3 сохранены два API-инстанса, HAProxy, PostgreSQL, Redis fallback, bearer-сессии, idempotency сообщений, курсоры, graceful shutdown и persistence/failover hooks. В lab4 добавлены: + +- `Attachment` со статусами `pending/ready/failed`, SHA-256, истинным MIME, размером и размерами изображения; +- таблицы `attachments` и `message_attachments`, same-chat foreign keys и deferred constraint «текст или вложение»; +- `attachment_ids` во всех слоях Message без нарушения старого JSON (`[]` по умолчанию); +- binary `POST`, metadata `GET` и авторизованный streaming `GET /content` без redirect; +- общий MinIO volume, idempotent bootstrap приватного bucket и отдельный API user с минимальной policy; +- readiness, зависящий от PostgreSQL и MinIO; +- one-shot reconciler с dry-run по умолчанию, `--apply`, grace period и условными DB updates. + +Upload возвращает `201 ready` только после последовательности: полное ограниченное чтение → проверка MIME/PNG/JPEG/пикселей и SHA-256 → insert `pending` → успешный S3 PUT → update `ready`. Ошибка PUT удаляет возможный объект и фиксирует `failed`. Ошибка ответа на финальный DB update считается неоднозначной: update мог уже закоммититься, поэтому объект не удаляется. API делает best-effort readback и либо восстанавливает успешный `201`, либо возвращает `503`, сохранив metadata+object для reconciler. Это исключает состояние `ready` без оригинала при потере ответа PostgreSQL. + +Лимиты применяются до неограниченной работы: body читается через `LimitReader(10 MiB + 1)`, `DecodeConfig` проверяет максимум 40 млн пикселей до полного `Decode`, а полный `Decode` проверяет поток/CRC/IDAT/SOS до S3 PUT. Filename не участвует в object key и проверяется на valid UTF-8/blank/path/C0+C1 control characters. Object key формируется сервером как `attachments/{chat_id}/{attachment_id}/original`. На реплику допускаются не более четырёх upload pipeline и одного полного decode одновременно; `GOMEMLIMIT=220MiB` ниже container limit 256 MiB. S3 operation timeout — 5 s, HTTP read/write timeout — 15 s, HAProxy server/client timeout — 30 s, DB query timeout задаётся конфигурацией. Компенсация сначала получает отдельный 1 s budget для DB transition и лишь затем отдельный 1 s budget для best-effort S3 delete. + +Публичный `/health/ready` проверяет PostgreSQL и MinIO и возвращает контрактный JSON `503`, если зависимость недоступна. HAProxy использует отдельный внутренний `/health/lb`, отражающий жизнеспособность/drain процесса, поэтому при общем отказе MinIO запросы не заменяются generic HTML-ошибкой балансировщика, а доходят до API и получают согласованный `503 unavailable`. + +Соответствие ключевых переменных: `PUBLIC_PORT` — host port HAProxy; `APP_INTERNAL_PORT` — порт API в сети Compose; `DATABASE_URL_FILE`, `CURSOR_SIGNING_KEY_FILE`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` — Docker secrets; `S3_ENDPOINT=http://minio:9000`; `S3_BUCKET=messenger-attachments`; `S3_OPERATION_TIMEOUT=5s`. + +## Механизм согласованности и трек 5 + +Гипотеза: периодический inventory PostgreSQL и MinIO с консервативным grace period способен повторяемо исправлять безопасные orphan/missing состояния, не удаляя свежие объекты и вложения, уже связанные с сообщениями. + +Reconciler сравнивает все `attachments/` objects с метаданными: + +- старый object без metadata удаляется только в apply; +- старый `pending` очищает object и условно становится `failed`; +- старый `ready` без object условно становится `failed`, только если не используется сообщением; +- object у старого `failed` удаляется, только если запись не используется; +- свежие записи/objects игнорируются, а referenced missing помечается `manual_repair_required` без мутации; +- SQL update берёт row lock, затем отдельным statement на свежем READ COMMITTED snapshot повторно проверяет status, cutoff и отсутствие ссылки; concurrent message/reconciler race либо создаёт ссылку, либо переводит attachment в failed, но не оба состояния одновременно; +- object stale-pending удаляется только после успешного условного DB transition, поэтому проигранная гонка приводит к `skipped_concurrent_change`, а не к потере готового оригинала. + +Команды оператора: + +```sh +make reconcile-dry-run RECONCILE_GRACE=15m +# Просмотреть JSON; затем при согласии: +make reconcile-apply RECONCILE_GRACE=15m +``` ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload/message/download/checksum/outsider 404 | [evidence/lab4-results.md](evidence/lab4-results.md) | +| Границы/полная декодировка | `make test-attachment-boundaries` | Ровно 10 MiB и 40 млн пикселей приняты; truncated/corrupt-after-64-KiB отклонены 415; whitespace/C1 filename — 400 | [tests/attachment_boundaries.py](tests/attachment_boundaries.py) | +| Failover API | `make test-failover` с указанными в evidence env | 142 успешных записи, 0 ошибок, recovery 2.771 s; файл читается другой репликой | [evidence/lab4-results.md](evidence/lab4-results.md) | +| Restart и volumes | `make test-persistence` | Сессия, метаданные и bytes пережили restart PostgreSQL/Redis/MinIO/API/frontend/HAProxy | [scripts/restart.sh](scripts/restart.sh) | +| Файловые негативные тесты | `go test ./...` | oversized bytes/pixels, invalid IDAT, truncated PNG/JPEG, corruption after 64 KiB, MIME substitution, unsafe filename, interrupted body, foreign attachment/chat | [internal/application/attachments_test.go](internal/application/attachments_test.go) | +| Неоднозначный DB COMMIT | `go test ./internal/application -count=1` | Потеря ответа после commit восстанавливается readback; при одновременном read outage ready metadata сохраняет оригинал | [internal/application/attachments_test.go](internal/application/attachments_test.go) | +| Реальный отказ MinIO | `make test-storage-outage` | readiness 503 за 2.004 s, upload 503 за 6.016 s; после recovery прежний original неизменен | [tests/storage_outage.py](tests/storage_outage.py) | +| RAM при concurrency ≥10 | `make test-attachment-memory` | 12 × 6,293,028 B, 12/12 success; пики 31,216,107 и 26,214,400 B | [tests/attachment_memory.py](tests/attachment_memory.py) | +| Private bucket / API policy | `make test-minio-policy` | Root secrets/env отсутствуют в API; limited user не создаёт bucket; anonymous GET запрещён | [tests/minio_policy.sh](tests/minio_policy.sh) | +| Reconciler dry-run/apply/repeat | `make test-reconcile` | 4 issues; 2 objects deleted; 2 metadata failed; repeat 0 mutations; referenced защищён | [tests/reconcile_integration.py](tests/reconcile_integration.py) | +| Стоимость обхода | `make test-reconcile-scale` | 100 objects: 9 ms; 1,000 objects: 38 ms core duration | [tests/reconcile_scale.py](tests/reconcile_scale.py) | +| Static/race gates | `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS | [evidence/lab4-results.md](evidence/lab4-results.md) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Memory experiment: один прогон, 12 concurrent uploads, каждый PNG 2048×1024 и 6,293,028 bytes; суммарно 75,516,336 bytes. HAProxy распределил запросы по двум репликам. Docker stats опрашивался во время прогона. 12/12 завершились за 0.599 s, наблюдаемые пики — 29.8 и 25.0 MiB, существенно ниже container limit 256 MiB. После измерения отдельный chat prefix и синтетические DB rows удалены. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Reconciler integration: в отдельном namespace созданы old orphan object, old pending metadata+object, unreferenced ready metadata без object и referenced ready metadata без object. Dry-run нашёл четыре состояния и не мутировал данные. Apply удалил два objects и перевёл две записи в `failed`. Повторный apply сделал ноль мутаций; referenced missing остался ручной задачей. Синтетические данные удалены в `finally`. + +Traversal experiment использовал один и тот же стенд после прогрева, один прогон на размер, отдельный prefix и однобайтовые objects. 100 injected orphans: 109 objects total, 9 ms core и 1,978 ms с запуском one-shot container. 1,000: 1,009 total, 38 ms core и 2,019 ms end-to-end. Это подтверждает повторяемость методики и ожидаемую близкую к линейной стоимость лишь на малом локальном масштабе. + +| Воздействие | Ожидание | Наблюдение | Потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Stop одной API-реплики | Вторая продолжает запись/чтение общего файла | recovery 2.771 s, 142 writes, 0 errors | Нет | +| Restart общего стенда | Named volumes сохраняют DB и S3 | persistence PASS | Нет | +| S3 PUT error | 503, metadata failed, object отсутствует | Unit injection PASS | Нет ready-данных | +| MinIO недоступен | readiness/upload дают JSON 503 до API/LB timeout | 2.004 s / 6.016 s, реальный stop/recovery PASS | Старый original неизменен | +| DB finalize response потерян после COMMIT | Readback возвращает ready; при DB outage object не удаляется | Два unit fault-теста PASS | Ready не остаётся без original | +| Orphan/missing/stale состояния | Dry-run, безопасный apply, repeat | Integration PASS | Referenced missing не изменён | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Inventory materializes metadata и object list в RAM; для большого bucket нужны pagination/checkpoints и пакетная выдача issues. +- Один reconciler предполагается запускать оператором; перед scheduler нужны PostgreSQL advisory lock/lease и метрики длительности/ошибок. +- `referenced_missing_object` намеренно не чинится автоматически: требуется восстановление bytes из backup или операторское решение. +- Один-node MinIO и локальные измерения не доказывают HA самого object storage; это учебный стенд. +- Experiment содержит по одному измерению на размер без статистических повторов; для production нужны warm-up, серии, p50/p95/p99, CPU/IO и существенно больший inventory. +- Оригинал проверяется SHA-256 при upload и клиентом в smoke; каждый download не пересчитывает hash на лету. diff --git a/cmd/reconciler/main.go b/cmd/reconciler/main.go new file mode 100644 index 0000000..97c3097 --- /dev/null +++ b/cmd/reconciler/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "time" + + "mtusi2026lab4/internal/adapter/objectstore" + "mtusi2026lab4/internal/adapter/postgres" + "mtusi2026lab4/internal/config" + "mtusi2026lab4/internal/database" + "mtusi2026lab4/internal/reconcile" +) + +func main() { + apply := flag.Bool("apply", false, "apply safe repairs; default is dry-run") + gracePeriod := flag.Duration("grace-period", 15*time.Minute, + "ignore metadata and objects newer than this duration") + timeout := flag.Duration("timeout", 2*time.Minute, "maximum reconciliation runtime") + flag.Parse() + + if *gracePeriod <= 0 || *timeout <= 0 { + log.Fatal("[FATAL] grace-period and timeout must be positive") + } + cfg, err := config.Load() + if err != nil { + log.Fatalf("[FATAL] Invalid configuration: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + databasePool, err := database.Open(ctx, cfg) + if err != nil { + log.Fatalf("[FATAL] PostgreSQL initialization failed: %v", err) + } + defer databasePool.Close() + metadataStore := postgres.NewPostgreSQLStore(databasePool, cfg.DatabaseQueryTimeout) + + objectStore, err := objectstore.NewMinIOStore( + cfg.S3Endpoint, + cfg.S3Region, + cfg.S3Bucket, + cfg.S3AccessKeyID, + cfg.S3SecretAccessKey, + cfg.S3OperationTimeout, + ) + if err != nil { + log.Fatalf("[FATAL] Invalid object storage configuration: %v", err) + } + if err = objectStore.Ping(ctx); err != nil { + log.Fatalf("[FATAL] Object storage unavailable: %v", err) + } + + reconciler, err := reconcile.New(metadataStore, objectStore) + if err != nil { + log.Fatalf("[FATAL] Reconciler initialization failed: %v", err) + } + result, err := reconciler.Run(ctx, *gracePeriod, *apply) + if err != nil { + log.Fatalf("[FATAL] Reconciliation failed: %v", err) + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err = encoder.Encode(result); err != nil { + log.Fatalf("[FATAL] Encode reconciliation result: %v", err) + } + if !*apply && len(result.Issues) > 0 { + fmt.Fprintln(os.Stderr, "dry-run found reconciliation issues; rerun with --apply after review") + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 694e0da..ab0603f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -11,6 +11,8 @@ import ( "time" cacheadapter "mtusi2026lab4/internal/adapter/cache" + "mtusi2026lab4/internal/adapter/health" + "mtusi2026lab4/internal/adapter/objectstore" "mtusi2026lab4/internal/adapter/postgres" redisadapter "mtusi2026lab4/internal/adapter/redis" "mtusi2026lab4/internal/application" @@ -43,6 +45,25 @@ func main() { databasePool, cfg.DatabaseQueryTimeout, ) + minioStore, err := objectstore.NewMinIOStore( + cfg.S3Endpoint, + cfg.S3Region, + cfg.S3Bucket, + cfg.S3AccessKeyID, + cfg.S3SecretAccessKey, + cfg.S3OperationTimeout, + ) + if err != nil { + log.Fatalf("[FATAL] Invalid object storage configuration: %v", err) + } + storageContext, storageCancel := context.WithTimeout( + context.Background(), + cfg.S3OperationTimeout, + ) + if err = minioStore.Ping(storageContext); err != nil { + log.Printf("[WARN] Object storage unavailable at startup: %v", err) + } + storageCancel() var applicationStore application.Repository = postgresStore @@ -90,6 +111,7 @@ func main() { cursorCodec, cfg.SessionTTL, application.WithIdempotencyTTL(cfg.IdempotencyTTL), + application.WithObjectStore(minioStore), ) if err != nil { log.Fatalf("[FATAL] Application initialization failed: %v", err) @@ -98,7 +120,7 @@ func main() { handler := handlers.NewHandler( service, handlers.WithReadiness( - databasePool, + health.NewMultiChecker(databasePool, minioStore), cfg.ReadinessTimeout, ), handlers.WithSessionCookieSecure(cfg.SessionCookieSecure), @@ -107,6 +129,7 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("GET /health/live", handler.HealthLive) + mux.HandleFunc("GET /health/lb", handler.HealthLive) mux.HandleFunc("GET /health/ready", handler.HealthReady) mux.HandleFunc( @@ -150,6 +173,18 @@ func main() { "GET /api/v1/chats/{chat_id}/messages", requireAuth(handler.GetMessages), ) + mux.HandleFunc( + "POST /api/v1/chats/{chat_id}/attachments", + requireAuth(handler.UploadAttachment), + ) + mux.HandleFunc( + "GET /api/v1/attachments/{attachment_id}", + requireAuth(handler.GetAttachment), + ) + mux.HandleFunc( + "GET /api/v1/attachments/{attachment_id}/content", + requireAuth(handler.DownloadAttachment), + ) var rootHandler http.Handler = mux rootHandler = middleware.Recovery(rootHandler) diff --git a/compose.yaml b/compose.yaml index 9969ecb..164bd2c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,7 +5,7 @@ services: dockerfile: infrastructure/haproxy/Dockerfile image: mtusi2026lab4-load-balancer ports: - - "127.0.0.1:${APP_PORT:-8080}:8080" + - "127.0.0.1:${PUBLIC_PORT:-8080}:8080" depends_on: api: condition: service_healthy @@ -128,13 +128,84 @@ services: cpus: "0.25" memory: 64M + minio: + build: + context: . + dockerfile: infrastructure/minio/Dockerfile + target: server + args: + MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z + image: mtusi2026lab4-minio:RELEASE.2025-10-15T17-29-55Z + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - minio_data:/data + expose: + - "9000" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + minio-bootstrap: + build: + context: . + dockerfile: infrastructure/minio/Dockerfile + target: client + args: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + image: mtusi2026lab4-mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: ["/bin/sh", "-ec"] + command: + - | + root_user="$$(cat /run/secrets/minio_root_user)" + root_password="$$(cat /run/secrets/minio_root_password)" + api_user="$$(cat /run/secrets/s3_access_key_id)" + api_password="$$(cat /run/secrets/s3_secret_access_key)" + mc alias set local http://minio:9000 "$$root_user" "$$root_password" --api S3v4 + mc mb --ignore-existing "local/$${S3_BUCKET}" + mc anonymous set none "local/$${S3_BUCKET}" + sed "s/__BUCKET__/$${S3_BUCKET}/g" /config/api-policy.template.json > /tmp/api-policy.json + mc admin policy create local messenger-api /tmp/api-policy.json + if ! mc admin user info local "$$api_user" >/dev/null 2>&1; then + mc admin user add local "$$api_user" "$$api_password" + fi + mc admin policy attach local messenger-api --user "$$api_user" + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infrastructure/minio/api-policy.template.json:/config/api-policy.template.json:ro + depends_on: + minio: + condition: service_healthy + restart: "no" + api: build: context: . dockerfile: Dockerfile environment: APP_HOST: ${APP_HOST:-0.0.0.0} - APP_PORT: ${APP_PORT:-8080} + APP_PORT: ${APP_INTERNAL_PORT:-8080} APP_ENV: ${APP_ENV:-lab} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} @@ -148,6 +219,13 @@ services: CACHE_OPERATION_TIMEOUT: ${CACHE_OPERATION_TIMEOUT:-250ms} CACHE_POOL_SIZE: ${CACHE_POOL_SIZE:-10} + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key} + S3_OPERATION_TIMEOUT: ${S3_OPERATION_TIMEOUT:-5s} + DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url} DB_MAX_CONNS: ${DB_MAX_CONNS:-10} @@ -161,8 +239,11 @@ services: READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s} SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s} SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s} + GOMEMLIMIT: ${API_GO_MEMORY_LIMIT:-220MiB} secrets: - database_url + - s3_access_key_id + - s3_secret_access_key - cursor_signing_key expose: - "8080" @@ -173,6 +254,8 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + minio-bootstrap: + condition: service_completed_successfully deploy: replicas: ${API_REPLICAS:-2} resources: @@ -240,6 +323,7 @@ services: volumes: postgres_data: redis_insight_data: + minio_data: postgres_restore_data: secrets: @@ -249,3 +333,11 @@ secrets: file: ./secrets/database_url.txt cursor_signing_key: file: ./secrets/cursor_signing_key.txt + minio_root_user: + file: ./secrets/minio_root_user.txt + minio_root_password: + file: ./secrets/minio_root_password.txt + s3_access_key_id: + file: ./secrets/s3_access_key_id.txt + s3_secret_access_key: + file: ./secrets/s3_secret_access_key.txt diff --git a/evidence/lab4-results.md b/evidence/lab4-results.md new file mode 100644 index 0000000..ce92d41 --- /dev/null +++ b/evidence/lab4-results.md @@ -0,0 +1,19 @@ +# Lab 4 verification summary + +Date: 2026-10-29. Environment: local Docker Compose on WSL2; two API replicas; PostgreSQL 16.15; Redis 8.10.2; MinIO release `RELEASE.2025-10-15T17-29-55Z` built from the official source module. + +| Check | Command | Observed result | +| --- | --- | --- | +| Public contract | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload, attachment-only message, authorized byte download, checksum/content type and outsider 404 | +| Exact boundaries / decode | `BASE_URL=http://127.0.0.1:18080 make test-attachment-boundaries` | Exact 10 MiB and 40,000,000 pixels accepted; truncated/corrupted-after-64-KiB PNG rejected 415; blank/C1 filename rejected 400 | +| One-replica failover | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/stop-one.sh make test-failover` | 2 instances before failure; 142 successful writes; 0 failed attempts; first recovery 2.771 s; attachment unchanged | +| Full persistence restart | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/restart.sh make test-persistence` | PASS after restarting PostgreSQL, Redis, MinIO, both API replicas, frontend and HAProxy; existing session, metadata and bytes survived | +| Real MinIO outage | `BASE_URL=http://127.0.0.1:18080 make test-storage-outage` | Public readiness returned JSON 503 in 2.004 s, upload returned JSON 503 in 6.016 s; after recovery the 111,817-byte previous original was byte-identical | +| Private bucket / least privilege | `make test-minio-policy` | API containers had no MinIO root env/secrets; limited credentials could put/read/delete only in configured bucket; creating another bucket and anonymous GET failed | +| Concurrent upload memory | `BASE_URL=http://127.0.0.1:18080 make test-attachment-memory` | 12/12 successful 6,293,028-byte PNG uploads (75,516,336 bytes total) in 0.599 s; peak API memory 31,216,107 and 26,214,400 bytes | +| Reconciler behavior | `make test-reconcile` | Dry-run found 4 injected issues; apply deleted 2 objects and marked 2 attachments failed; repeat made 0 mutations; 1 referenced missing attachment remained protected/manual | +| Reconciler traversal | `make test-reconcile-scale` | 100 orphan objects: 109 objects scanned, 9 ms core / 1,978 ms including container startup. 1,000 orphans: 1,009 scanned, 38 ms core / 2,019 ms including startup | +| Ambiguous DB COMMIT fault | `go test ./internal/application -count=1` | Successful finalize with lost response is recovered by readback; if DB remains unavailable, ready metadata keeps its object and returns an honest error | +| Go quality gates | `go test ./...`; `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS | + +The measurements are single local runs, not production capacity claims. The reconciler currently materializes both inventories in memory, so its measured near-linear small-scale cost must be re-evaluated before using it with a large bucket. diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 9a227eb..1e95a48 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -143,7 +143,12 @@ export const api = { ); }, - sendMessage(chatID: string, text: string, idempotencyKey: string): Promise { + sendMessage( + chatID: string, + text: string, + idempotencyKey: string, + attachmentIDs: string[] = [], + ): Promise { return request( `${browserApi}/chats/${encodeURIComponent(chatID)}/messages`, { @@ -151,7 +156,7 @@ export const api = { headers: { "Idempotency-Key": idempotencyKey, }, - body: { text }, + body: { text, attachment_ids: attachmentIDs }, }, ); }, diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 21283d8..0f7110a 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -24,6 +24,7 @@ export type Message = { chat_id: string; sender_id: string; text: string; + attachment_ids: string[]; created_at: string; }; diff --git a/go.mod b/go.mod index 5e593e5..9af204e 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,31 @@ go 1.27.1 require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 github.com/redis/go-redis/v9 v9.23.0 golang.org/x/crypto v0.57.0 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index 409359b..63ca12b 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,9 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -15,16 +18,42 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= @@ -33,6 +62,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= @@ -40,4 +71,7 @@ golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infrastructure/haproxy/haproxy.cfg b/infrastructure/haproxy/haproxy.cfg index 3a4f34a..263dc41 100644 --- a/infrastructure/haproxy/haproxy.cfg +++ b/infrastructure/haproxy/haproxy.cfg @@ -43,7 +43,7 @@ frontend public backend api_backend balance leastconn option httpchk - http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api http-check expect status 200 default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none server-template api 8 api:8080 @@ -55,7 +55,7 @@ backend message_backend option http-buffer-request retries 3 retry-on conn-failure empty-response response-timeout 502 503 504 - http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api http-check expect status 200 default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none server-template api 8 api:8080 diff --git a/infrastructure/minio/Dockerfile b/infrastructure/minio/Dockerfile new file mode 100644 index 0000000..19a630a --- /dev/null +++ b/infrastructure/minio/Dockerfile @@ -0,0 +1,40 @@ +FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS minio-builder + +ARG MINIO_VERSION=RELEASE.2025-10-15T17-29-55Z +RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/minio@${MINIO_VERSION} + +FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS mc-builder + +ARG MC_VERSION=RELEASE.2025-08-13T08-35-41Z +RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/mc@${MC_VERSION} + +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS server + +RUN apk add --no-cache ca-certificates wget && \ + addgroup -g 1000 minio && \ + adduser -u 1000 -G minio -s /sbin/nologin -D minio && \ + mkdir -p /data && \ + chown minio:minio /data + +COPY --from=minio-builder /out/minio /usr/local/bin/minio + +USER minio:minio +VOLUME ["/data"] +EXPOSE 9000 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=20 \ + CMD wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1 + +ENTRYPOINT ["/usr/local/bin/minio"] +CMD ["server", "/data", "--address", ":9000"] + +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS client + +RUN apk add --no-cache ca-certificates && \ + addgroup -g 1000 minio && \ + adduser -u 1000 -G minio -s /sbin/nologin -D minio + +COPY --from=mc-builder /out/mc /usr/local/bin/mc + +USER minio:minio +ENTRYPOINT ["/usr/local/bin/mc"] diff --git a/infrastructure/minio/api-policy.template.json b/infrastructure/minio/api-policy.template.json new file mode 100644 index 0000000..43f2254 --- /dev/null +++ b/infrastructure/minio/api-policy.template.json @@ -0,0 +1,26 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::__BUCKET__" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::__BUCKET__/*" + ] + } + ] +} diff --git a/internal/adapter/health/multi.go b/internal/adapter/health/multi.go new file mode 100644 index 0000000..bfb402c --- /dev/null +++ b/internal/adapter/health/multi.go @@ -0,0 +1,33 @@ +package health + +import ( + "context" + "fmt" +) + +type Checker interface { + Ping(ctx context.Context) error +} + +type MultiChecker struct { + checkers []Checker +} + +func NewMultiChecker(checkers ...Checker) *MultiChecker { + filtered := make([]Checker, 0, len(checkers)) + for _, checker := range checkers { + if checker != nil { + filtered = append(filtered, checker) + } + } + return &MultiChecker{checkers: filtered} +} + +func (checker *MultiChecker) Ping(ctx context.Context) error { + for index, dependency := range checker.checkers { + if err := dependency.Ping(ctx); err != nil { + return fmt.Errorf("dependency %d: %w", index, err) + } + } + return nil +} diff --git a/internal/adapter/memory/repository.go b/internal/adapter/memory/repository.go index 7748bf6..11ef177 100644 --- a/internal/adapter/memory/repository.go +++ b/internal/adapter/memory/repository.go @@ -166,6 +166,15 @@ func (r *MemoryRepository) CreateMessageIdempotently( if _, err := r.legacy.GetChatByID(write.ChatID, write.ActorID); err != nil { return domain.Message{}, err } + for _, attachmentID := range write.AttachmentIDs { + attachment, err := r.legacy.GetAttachmentByID(attachmentID) + if err != nil || attachment.ChatID != write.ChatID { + return domain.Message{}, ErrNotFound + } + if attachment.Status != domain.AttachmentStatusReady { + return domain.Message{}, application.ErrAttachmentNotReady + } + } if stored, exists := r.idempotencyByScope[scope]; exists { if stored.expiresAt.After(now) { @@ -177,7 +186,12 @@ func (r *MemoryRepository) CreateMessageIdempotently( delete(r.idempotencyByScope, scope) } - message, err := r.legacy.CreateMessage(write.ChatID, write.ActorID, write.Text) + message, err := r.legacy.CreateMessage( + write.ChatID, + write.ActorID, + write.Text, + write.AttachmentIDs, + ) if err != nil { return domain.Message{}, err } @@ -277,6 +291,74 @@ func (r *MemoryRepository) DeleteSession( return nil } +func (r *MemoryRepository) CreateAttachment( + ctx context.Context, + attachment domain.Attachment, +) (domain.Attachment, error) { + if err := ctx.Err(); err != nil { + return domain.Attachment{}, err + } + if err := r.legacy.PutAttachment(attachment); err != nil { + return domain.Attachment{}, err + } + return attachment, nil +} + +func (r *MemoryRepository) MarkAttachmentReady( + ctx context.Context, + attachmentID string, + updatedAt time.Time, +) (domain.Attachment, error) { + if err := ctx.Err(); err != nil { + return domain.Attachment{}, err + } + return r.legacy.UpdateAttachmentStatus( + attachmentID, + domain.AttachmentStatusReady, + nil, + updatedAt, + ) +} + +func (r *MemoryRepository) MarkAttachmentFailed( + ctx context.Context, + attachmentID string, + errorCode string, + updatedAt time.Time, +) error { + if err := ctx.Err(); err != nil { + return err + } + _, err := r.legacy.UpdateAttachmentStatus( + attachmentID, + domain.AttachmentStatusFailed, + &errorCode, + updatedAt, + ) + return err +} + +func (r *MemoryRepository) GetAttachment( + ctx context.Context, + attachmentID string, + callerID string, +) (domain.Attachment, error) { + if err := ctx.Err(); err != nil { + return domain.Attachment{}, err + } + attachment, err := r.legacy.GetAttachmentByID(attachmentID) + if err != nil { + return domain.Attachment{}, err + } + if _, err = r.legacy.GetChatByID(attachment.ChatID, callerID); err != nil { + return domain.Attachment{}, err + } + if attachment.Variants == nil { + attachment.Variants = make([]domain.AttachmentVariant, 0) + } + return attachment, nil +} + func memoryTokenHashKey(tokenHash []byte) ([32]byte, error) { var key [32]byte diff --git a/internal/adapter/memory/store.go b/internal/adapter/memory/store.go index 217bf21..c239384 100644 --- a/internal/adapter/memory/store.go +++ b/internal/adapter/memory/store.go @@ -20,6 +20,8 @@ type MemoryStore struct { chatsByID map[string]domain.Chat //key = chat UUID messagesByChatID map[string][]domain.Message // key = chat UUID, value = слайс сообщений + + attachmentsByID map[string]domain.Attachment } func NewMemoryStore() *MemoryStore { @@ -28,6 +30,7 @@ func NewMemoryStore() *MemoryStore { usersByUsername: make(map[string]string), chatsByID: make(map[string]domain.Chat), messagesByChatID: make(map[string][]domain.Message), + attachmentsByID: make(map[string]domain.Attachment), } } @@ -177,12 +180,86 @@ func (s *MemoryStore) GetChatsForUser( return items, end < len(chats), nil } +// PutAttachment stores metadata for adapter and application tests. Binary +// content intentionally does not belong to this in-memory metadata store. +func (s *MemoryStore) PutAttachment(attachment domain.Attachment) error { + s.mu.Lock() + defer s.mu.Unlock() + + if _, exists := s.attachmentsByID[attachment.ID]; exists { + return ErrConflict + } + + chat, exists := s.chatsByID[attachment.ChatID] + if !exists { + return ErrNotFound + } + + ownerIsMember := false + for _, memberID := range chat.MemberIDs { + if memberID == attachment.OwnerID { + ownerIsMember = true + break + } + } + if !ownerIsMember { + return ErrNotFound + } + + s.attachmentsByID[attachment.ID] = attachment + return nil +} + +func (s *MemoryStore) GetAttachmentByID( + attachmentID string, +) (domain.Attachment, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + attachment, exists := s.attachmentsByID[attachmentID] + if !exists { + return domain.Attachment{}, ErrNotFound + } + return attachment, nil +} + +func (s *MemoryStore) UpdateAttachmentStatus( + attachmentID string, + status domain.AttachmentStatus, + errorCode *string, + updatedAt time.Time, +) (domain.Attachment, error) { + s.mu.Lock() + defer s.mu.Unlock() + + attachment, exists := s.attachmentsByID[attachmentID] + if !exists { + return domain.Attachment{}, ErrNotFound + } + if attachment.Status != domain.AttachmentStatusPending { + return domain.Attachment{}, ErrConflict + } + attachment.Status = status + attachment.ErrorCode = errorCode + attachment.UpdatedAt = updatedAt + if attachment.Variants == nil { + attachment.Variants = make([]domain.AttachmentVariant, 0) + } + s.attachmentsByID[attachmentID] = attachment + return attachment, nil +} + // ---Сообщения--- func (s *MemoryStore) CreateMessage( chatID, senderID, text string, + attachmentLists ...[]string, ) (domain.Message, error) { messageID := uuid.NewString() + attachmentIDs := make([]string, 0) + if len(attachmentLists) > 0 { + attachmentIDs = append(attachmentIDs, attachmentLists[0]...) + } s.mu.Lock() defer s.mu.Unlock() @@ -216,11 +293,12 @@ func (s *MemoryStore) CreateMessage( } message := domain.Message{ - ID: messageID, - ChatID: chatID, - SenderID: senderID, - Text: text, - CreatedAt: createdAt, + ID: messageID, + ChatID: chatID, + SenderID: senderID, + Text: text, + AttachmentIDs: attachmentIDs, + CreatedAt: createdAt, } s.messagesByChatID[chatID] = append(messages, message) @@ -282,6 +360,12 @@ func (s *MemoryStore) GetMessages( // копируем только возвращаемую страницу, а не всю историю, изменяя внутренний срез items := make([]domain.Message, end-start) - copy(items, messages[start:end]) + for index, message := range messages[start:end] { + message.AttachmentIDs = append( + make([]string, 0, len(message.AttachmentIDs)), + message.AttachmentIDs..., + ) + items[index] = message + } return items, end < len(messages), nil } diff --git a/internal/adapter/objectstore/minio.go b/internal/adapter/objectstore/minio.go new file mode 100644 index 0000000..2ce5407 --- /dev/null +++ b/internal/adapter/objectstore/minio.go @@ -0,0 +1,201 @@ +package objectstore + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + "strings" + "time" + + "mtusi2026lab4/internal/application" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type MinIOStore struct { + client *minio.Client + bucket string + operationTimeout time.Duration +} + +type StoredObject struct { + Key string + SizeBytes int64 + LastModified time.Time +} + +func NewMinIOStore( + endpoint string, + region string, + bucket string, + accessKeyID string, + secretAccessKey string, + operationTimeout time.Duration, +) (*MinIOStore, error) { + parsedEndpoint, err := url.Parse(endpoint) + if err != nil || parsedEndpoint.Host == "" { + return nil, fmt.Errorf("S3 endpoint must be an absolute HTTP(S) URL") + } + if parsedEndpoint.Scheme != "http" && parsedEndpoint.Scheme != "https" { + return nil, fmt.Errorf("S3 endpoint scheme must be http or https") + } + if parsedEndpoint.Path != "" && parsedEndpoint.Path != "/" { + return nil, fmt.Errorf("S3 endpoint must not contain a path") + } + if parsedEndpoint.RawQuery != "" || parsedEndpoint.Fragment != "" || parsedEndpoint.User != nil { + return nil, fmt.Errorf("S3 endpoint must not contain credentials, query or fragment") + } + if strings.TrimSpace(bucket) == "" || strings.TrimSpace(accessKeyID) == "" || + strings.TrimSpace(secretAccessKey) == "" { + return nil, fmt.Errorf("S3 bucket and credentials are required") + } + if operationTimeout <= 0 { + return nil, fmt.Errorf("S3 operation timeout must be positive") + } + + client, err := minio.New(parsedEndpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: parsedEndpoint.Scheme == "https", + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("create MinIO client: %w", err) + } + return &MinIOStore{client: client, bucket: bucket, operationTimeout: operationTimeout}, nil +} + +func (store *MinIOStore) Put(ctx context.Context, object application.ObjectPut) error { + operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout) + defer cancel() + upload, err := store.client.PutObject( + operationContext, + store.bucket, + object.Key, + object.Body, + object.SizeBytes, + minio.PutObjectOptions{ContentType: object.ContentType}, + ) + if err != nil { + return fmt.Errorf("put MinIO object: %w", err) + } + if upload.Size != object.SizeBytes { + return fmt.Errorf("put MinIO object: stored %d bytes, want %d", upload.Size, object.SizeBytes) + } + return nil +} + +func (store *MinIOStore) Get( + ctx context.Context, + key string, +) (application.ObjectContent, error) { + operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout) + stat, err := store.client.StatObject( + operationContext, + store.bucket, + key, + minio.StatObjectOptions{}, + ) + if err != nil { + cancel() + return application.ObjectContent{}, fmt.Errorf("stat MinIO object: %w", err) + } + object, err := store.client.GetObject( + operationContext, + store.bucket, + key, + minio.GetObjectOptions{}, + ) + if err != nil { + cancel() + return application.ObjectContent{}, fmt.Errorf("get MinIO object: %w", err) + } + firstByte := make([]byte, 1) + readCount, readError := object.Read(firstByte) + if readError != nil && readCount == 0 { + _ = object.Close() + cancel() + return application.ObjectContent{}, fmt.Errorf("read MinIO object: %w", readError) + } + if readCount != 1 { + _ = object.Close() + cancel() + return application.ObjectContent{}, fmt.Errorf("read MinIO object: expected non-empty object") + } + return application.ObjectContent{ + Body: &cancelingReadCloser{ + Reader: io.MultiReader(bytes.NewReader(firstByte), object), + Closer: object, + cancel: cancel, + }, + SizeBytes: stat.Size, + }, nil +} + +func (store *MinIOStore) List( + ctx context.Context, + prefix string, +) ([]StoredObject, error) { + operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout) + defer cancel() + + objects := make([]StoredObject, 0) + for object := range store.client.ListObjects( + operationContext, + store.bucket, + minio.ListObjectsOptions{Prefix: prefix, Recursive: true}, + ) { + if object.Err != nil { + return nil, fmt.Errorf("list MinIO objects: %w", object.Err) + } + objects = append(objects, StoredObject{ + Key: object.Key, + SizeBytes: object.Size, + LastModified: object.LastModified.UTC(), + }) + } + return objects, nil +} + +func (store *MinIOStore) Delete(ctx context.Context, key string) error { + operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout) + defer cancel() + if err := store.client.RemoveObject( + operationContext, + store.bucket, + key, + minio.RemoveObjectOptions{}, + ); err != nil { + return fmt.Errorf("delete MinIO object: %w", err) + } + return nil +} + +func (store *MinIOStore) Ping(ctx context.Context) error { + operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout) + defer cancel() + exists, err := store.client.BucketExists(operationContext, store.bucket) + if err != nil { + return fmt.Errorf("check MinIO bucket: %w", err) + } + if !exists { + return fmt.Errorf("MinIO bucket %q does not exist", store.bucket) + } + return nil +} + +type cancelingReadCloser struct { + io.Reader + io.Closer + cancel context.CancelFunc +} + +func (reader *cancelingReadCloser) Close() error { + err := reader.Closer.Close() + reader.cancel() + return err +} + +var _ application.ObjectStore = (*MinIOStore)(nil) diff --git a/internal/adapter/objectstore/minio_test.go b/internal/adapter/objectstore/minio_test.go new file mode 100644 index 0000000..204debd --- /dev/null +++ b/internal/adapter/objectstore/minio_test.go @@ -0,0 +1,37 @@ +package objectstore + +import ( + "testing" + "time" +) + +func TestNewMinIOStoreValidatesConfiguration(t *testing.T) { + testCases := []struct { + name string + endpoint string + bucket string + accessKey string + secretKey string + timeout time.Duration + wantError bool + }{ + {name: "valid HTTP", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second}, + {name: "valid HTTPS", endpoint: "https://s3.example.test", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second}, + {name: "relative endpoint", endpoint: "minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true}, + {name: "endpoint path", endpoint: "http://minio:9000/path", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true}, + {name: "embedded credentials", endpoint: "http://user:pass@minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true}, + {name: "empty bucket", endpoint: "http://minio:9000", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true}, + {name: "empty credentials", endpoint: "http://minio:9000", bucket: "attachments", timeout: time.Second, wantError: true}, + {name: "invalid timeout", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", wantError: true}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + _, err := NewMinIOStore(testCase.endpoint, "us-east-1", testCase.bucket, + testCase.accessKey, testCase.secretKey, testCase.timeout) + if (err != nil) != testCase.wantError { + t.Fatalf("NewMinIOStore() error = %v, wantError %v", err, testCase.wantError) + } + }) + } +} diff --git a/internal/adapter/postgres/attachments.go b/internal/adapter/postgres/attachments.go new file mode 100644 index 0000000..bfbc97e --- /dev/null +++ b/internal/adapter/postgres/attachments.go @@ -0,0 +1,306 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "mtusi2026lab4/internal/domain" + "mtusi2026lab4/internal/reconcile" + + "github.com/jackc/pgx/v5" +) + +func (s *PostgreSQLStore) CreateAttachment( + ctx context.Context, + attachment domain.Attachment, +) (domain.Attachment, error) { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + + err := s.pool.QueryRow( + queryContext, + ` + INSERT INTO attachments ( + id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, width, height, status, error_code, + created_at, updated_at + ) + VALUES ( + $1, $2, $3, $4, $5, $6, + $7, decode($8, 'hex'), $9, $10, $11, $12, + $13, $14 + ) + RETURNING created_at, updated_at + `, + attachment.ID, + attachment.ChatID, + attachment.OwnerID, + attachment.ObjectKey, + attachment.Filename, + attachment.ContentType, + attachment.SizeBytes, + attachment.SHA256, + attachment.Width, + attachment.Height, + attachment.Status, + attachment.ErrorCode, + attachment.CreatedAt, + attachment.UpdatedAt, + ).Scan(&attachment.CreatedAt, &attachment.UpdatedAt) + if err != nil { + return domain.Attachment{}, fmt.Errorf("insert attachment metadata: %w", err) + } + if attachment.Variants == nil { + attachment.Variants = make([]domain.AttachmentVariant, 0) + } + return attachment, nil +} + +func (s *PostgreSQLStore) MarkAttachmentReady( + ctx context.Context, + attachmentID string, + updatedAt time.Time, +) (domain.Attachment, error) { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + + var attachment domain.Attachment + var sha256Bytes []byte + err := s.pool.QueryRow( + queryContext, + ` + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = $2 + WHERE id = $1 AND status = 'pending' + RETURNING + id::text, chat_id::text, owner_id::text, object_key, + filename, content_type, size_bytes, sha256, width, height, + status, error_code, created_at, updated_at + `, + attachmentID, + updatedAt, + ).Scan( + &attachment.ID, + &attachment.ChatID, + &attachment.OwnerID, + &attachment.ObjectKey, + &attachment.Filename, + &attachment.ContentType, + &attachment.SizeBytes, + &sha256Bytes, + &attachment.Width, + &attachment.Height, + &attachment.Status, + &attachment.ErrorCode, + &attachment.CreatedAt, + &attachment.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return domain.Attachment{}, ErrConflict + } + if err != nil { + return domain.Attachment{}, fmt.Errorf("mark attachment ready: %w", err) + } + attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes) + attachment.Variants = make([]domain.AttachmentVariant, 0) + return attachment, nil +} + +func (s *PostgreSQLStore) MarkAttachmentFailed( + ctx context.Context, + attachmentID string, + errorCode string, + updatedAt time.Time, +) error { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + + commandTag, err := s.pool.Exec( + queryContext, + ` + UPDATE attachments + SET status = 'failed', error_code = $2, updated_at = $3 + WHERE id = $1 AND status = 'pending' + `, + attachmentID, + errorCode, + updatedAt, + ) + if err != nil { + return fmt.Errorf("mark attachment failed: %w", err) + } + if commandTag.RowsAffected() != 1 { + return ErrConflict + } + return nil +} + +func (s *PostgreSQLStore) GetAttachment( + ctx context.Context, + attachmentID string, + callerID string, +) (domain.Attachment, error) { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + + var attachment domain.Attachment + var sha256Bytes []byte + err := s.pool.QueryRow( + queryContext, + ` + SELECT + a.id::text, a.chat_id::text, a.owner_id::text, a.object_key, + a.filename, a.content_type, a.size_bytes, a.sha256, + a.width, a.height, a.status, a.error_code, + a.created_at, a.updated_at + FROM attachments AS a + JOIN chat_members AS cm + ON cm.chat_id = a.chat_id AND cm.user_id = $2 + WHERE a.id = $1 + `, + attachmentID, + callerID, + ).Scan( + &attachment.ID, + &attachment.ChatID, + &attachment.OwnerID, + &attachment.ObjectKey, + &attachment.Filename, + &attachment.ContentType, + &attachment.SizeBytes, + &sha256Bytes, + &attachment.Width, + &attachment.Height, + &attachment.Status, + &attachment.ErrorCode, + &attachment.CreatedAt, + &attachment.UpdatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return domain.Attachment{}, ErrNotFound + } + if err != nil { + return domain.Attachment{}, fmt.Errorf("get attachment metadata: %w", err) + } + attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes) + attachment.Variants = make([]domain.AttachmentVariant, 0) + return attachment, nil +} + +func (s *PostgreSQLStore) ListAttachmentInventory( + ctx context.Context, +) ([]reconcile.AttachmentRecord, error) { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + + rows, err := s.pool.Query( + queryContext, + ` + SELECT + a.id::text, + a.object_key, + a.status, + a.updated_at, + EXISTS ( + SELECT 1 + FROM message_attachments AS ma + WHERE ma.attachment_id = a.id + ) AS referenced + FROM attachments AS a + ORDER BY a.id + `, + ) + if err != nil { + return nil, fmt.Errorf("query attachment inventory: %w", err) + } + defer rows.Close() + + records := make([]reconcile.AttachmentRecord, 0) + for rows.Next() { + var record reconcile.AttachmentRecord + if err = rows.Scan( + &record.ID, + &record.ObjectKey, + &record.Status, + &record.UpdatedAt, + &record.Referenced, + ); err != nil { + return nil, fmt.Errorf("scan attachment inventory: %w", err) + } + records = append(records, record) + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("iterate attachment inventory: %w", err) + } + return records, nil +} + +func (s *PostgreSQLStore) MarkAttachmentFailedIfUnreferenced( + ctx context.Context, + attachmentID string, + expectedStatus domain.AttachmentStatus, + updatedBefore time.Time, + errorCode string, + updatedAt time.Time, +) (bool, error) { + queryContext, cancel := s.withTimeout(ctx) + defer cancel() + transaction, err := s.pool.BeginTx(queryContext, pgx.TxOptions{}) + if err != nil { + return false, fmt.Errorf("begin attachment reconciliation transaction: %w", err) + } + defer func() { _ = transaction.Rollback(queryContext) }() + + var lockedAttachmentID string + err = transaction.QueryRow( + queryContext, + ` + SELECT id::text + FROM attachments + WHERE id = $1 + FOR UPDATE + `, + attachmentID, + ).Scan(&lockedAttachmentID) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("lock attachment during reconciliation: %w", err) + } + + // This is deliberately a second statement. Under READ COMMITTED it gets a + // fresh snapshot after the row lock was acquired, so a message transaction + // that linked this attachment while we waited is visible to NOT EXISTS. + commandTag, err := transaction.Exec( + queryContext, + ` + UPDATE attachments AS a + SET status = 'failed', error_code = $4, updated_at = $5 + WHERE a.id = $1 + AND a.status = $2 + AND a.updated_at <= $3 + AND NOT EXISTS ( + SELECT 1 + FROM message_attachments AS ma + WHERE ma.attachment_id = a.id + ) + `, + attachmentID, + expectedStatus, + updatedBefore, + errorCode, + updatedAt, + ) + if err != nil { + return false, fmt.Errorf("mark attachment failed during reconciliation: %w", err) + } + if err = transaction.Commit(queryContext); err != nil { + return false, fmt.Errorf("commit attachment reconciliation: %w", err) + } + return commandTag.RowsAffected() == 1, nil +} + +var _ reconcile.MetadataStore = (*PostgreSQLStore)(nil) diff --git a/internal/adapter/postgres/messages.go b/internal/adapter/postgres/messages.go index ae9f4c5..5071173 100644 --- a/internal/adapter/postgres/messages.go +++ b/internal/adapter/postgres/messages.go @@ -48,6 +48,59 @@ func (s *PostgreSQLStore) CreateMessageIdempotently( return domain.Message{}, ErrNotFound } + attachmentUUIDs := make([]uuid.UUID, len(write.AttachmentIDs)) + for index, attachmentID := range write.AttachmentIDs { + parsedID, parseErr := uuid.Parse(attachmentID) + if parseErr != nil { + return domain.Message{}, fmt.Errorf("parse validated attachment id: %w", parseErr) + } + attachmentUUIDs[index] = parsedID + } + + if len(attachmentUUIDs) > 0 { + rows, queryError := transaction.Query( + queryContext, + ` + SELECT id::text, status + FROM attachments + WHERE chat_id = $1 + AND id = ANY($2::uuid[]) + ORDER BY id + FOR UPDATE + `, + write.ChatID, + attachmentUUIDs, + ) + if queryError != nil { + return domain.Message{}, fmt.Errorf("lock message attachments: %w", queryError) + } + availableCount := 0 + readyCount := 0 + for rows.Next() { + var attachmentID string + var status domain.AttachmentStatus + if queryError = rows.Scan(&attachmentID, &status); queryError != nil { + rows.Close() + return domain.Message{}, fmt.Errorf("scan locked message attachment: %w", queryError) + } + availableCount++ + if status == domain.AttachmentStatusReady { + readyCount++ + } + } + queryError = rows.Err() + rows.Close() + if queryError != nil { + return domain.Message{}, fmt.Errorf("iterate locked message attachments: %w", queryError) + } + if availableCount != len(attachmentUUIDs) { + return domain.Message{}, ErrNotFound + } + if readyCount != len(attachmentUUIDs) { + return domain.Message{}, application.ErrAttachmentNotReady + } + } + _, err = transaction.Exec( queryContext, ` @@ -68,10 +121,11 @@ func (s *PostgreSQLStore) CreateMessageIdempotently( } message := domain.Message{ - ID: uuid.NewString(), - ChatID: write.ChatID, - SenderID: write.ActorID, - Text: write.Text, + ID: uuid.NewString(), + ChatID: write.ChatID, + SenderID: write.ActorID, + Text: write.Text, + AttachmentIDs: append([]string(nil), write.AttachmentIDs...), } var storedHash []byte @@ -141,7 +195,15 @@ func (s *PostgreSQLStore) CreateMessageIdempotently( chat_id::text, sender_id::text, text, - created_at + created_at, + COALESCE( + ARRAY( + SELECT attachment_id::text + FROM message_attachments + WHERE message_id = messages.id + ORDER BY position + ), ARRAY[]::text[] + ) FROM messages WHERE id = $1 `, @@ -152,6 +214,7 @@ func (s *PostgreSQLStore) CreateMessageIdempotently( &message.SenderID, &message.Text, &message.CreatedAt, + &message.AttachmentIDs, ) if err != nil { return domain.Message{}, fmt.Errorf("read idempotent message: %w", err) @@ -187,6 +250,28 @@ func (s *PostgreSQLStore) CreateMessageIdempotently( return domain.Message{}, fmt.Errorf("insert message: %w", err) } + if len(attachmentUUIDs) > 0 { + _, err = transaction.Exec( + queryContext, + ` + INSERT INTO message_attachments ( + message_id, + chat_id, + attachment_id, + position + ) + SELECT $1, $2, attachment_id, (ordinality - 1)::smallint + FROM unnest($3::uuid[]) WITH ORDINALITY AS input(attachment_id, ordinality) + `, + message.ID, + message.ChatID, + attachmentUUIDs, + ) + if err != nil { + return domain.Message{}, fmt.Errorf("link message attachments: %w", err) + } + } + if err = transaction.Commit(queryContext); err != nil { return domain.Message{}, fmt.Errorf("commit idempotent message: %w", err) } @@ -241,7 +326,15 @@ func (s *PostgreSQLStore) GetMessages( chat_id::text, sender_id::text, text, - created_at + created_at, + COALESCE( + ARRAY( + SELECT attachment_id::text + FROM message_attachments + WHERE message_id = messages.id + ORDER BY position + ), ARRAY[]::text[] + ) FROM messages WHERE chat_id = $1 ORDER BY @@ -261,7 +354,15 @@ func (s *PostgreSQLStore) GetMessages( chat_id::text, sender_id::text, text, - created_at + created_at, + COALESCE( + ARRAY( + SELECT attachment_id::text + FROM message_attachments + WHERE message_id = messages.id + ORDER BY position + ), ARRAY[]::text[] + ) FROM messages WHERE chat_id = $1 AND (created_at, id) > ($2, $3) @@ -296,6 +397,7 @@ func (s *PostgreSQLStore) GetMessages( &message.SenderID, &message.Text, &message.CreatedAt, + &message.AttachmentIDs, ); err != nil { return nil, false, fmt.Errorf( "scan message page: %w", diff --git a/internal/application/attachment_queries.go b/internal/application/attachment_queries.go new file mode 100644 index 0000000..0df02b1 --- /dev/null +++ b/internal/application/attachment_queries.go @@ -0,0 +1,65 @@ +package application + +import ( + "context" + "fmt" + "io" + + "mtusi2026lab4/internal/domain" +) + +type AttachmentDownload struct { + Body io.ReadCloser + ContentType string + Filename string + SizeBytes int64 +} + +func (s *Service) GetAttachment( + ctx context.Context, + attachmentID string, + actorID string, +) (domain.Attachment, error) { + return s.repository.GetAttachment(ctx, attachmentID, actorID) +} + +func (s *Service) DownloadAttachment( + ctx context.Context, + attachmentID string, + actorID string, + variant string, +) (AttachmentDownload, error) { + attachment, err := s.repository.GetAttachment(ctx, attachmentID, actorID) + if err != nil { + return AttachmentDownload{}, err + } + if attachment.Status != domain.AttachmentStatusReady { + return AttachmentDownload{}, ErrAttachmentNotReady + } + if variant == "" { + variant = "original" + } + if variant != "original" { + return AttachmentDownload{}, domain.ErrNotFound + } + if s.objectStore == nil { + return AttachmentDownload{}, ErrStorageUnavailable + } + + object, err := s.objectStore.Get(ctx, attachment.ObjectKey) + if err != nil { + return AttachmentDownload{}, fmt.Errorf("%w: get object", ErrStorageUnavailable) + } + if object.Body == nil || object.SizeBytes != attachment.SizeBytes { + if object.Body != nil { + _ = object.Body.Close() + } + return AttachmentDownload{}, fmt.Errorf("%w: object metadata mismatch", ErrStorageUnavailable) + } + return AttachmentDownload{ + Body: object.Body, + ContentType: attachment.ContentType, + Filename: attachment.Filename, + SizeBytes: attachment.SizeBytes, + }, nil +} diff --git a/internal/application/attachments.go b/internal/application/attachments.go new file mode 100644 index 0000000..a07693e --- /dev/null +++ b/internal/application/attachments.go @@ -0,0 +1,246 @@ +package application + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "os" + + "mtusi2026lab4/internal/domain" + + "github.com/google/uuid" +) + +const attachmentPutFailedCode = "object_store_put_failed" +const attachmentFinalizeFailedCode = "metadata_finalize_failed" + +func (s *Service) UploadAttachment( + ctx context.Context, + command UploadAttachmentCommand, +) (domain.Attachment, error) { + if s.objectStore == nil { + return domain.Attachment{}, ErrStorageUnavailable + } + if command.Body == nil { + return domain.Attachment{}, fmt.Errorf("%w: attachment body is required", ErrInvalidInput) + } + if err := domain.ValidateAttachmentFilename(command.Filename); err != nil { + return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidInput, err) + } + if err := domain.ValidateAttachmentContentType(command.DeclaredContentType); err != nil { + return domain.Attachment{}, fmt.Errorf("%w: %v", ErrUnsupportedMediaType, err) + } + if _, err := s.repository.GetChatByID(ctx, command.ChatID, command.ActorID); err != nil { + return domain.Attachment{}, err + } + if err := acquireAttachmentSlot(ctx, s.attachmentUploadSlots); err != nil { + return domain.Attachment{}, fmt.Errorf("%w: wait for upload capacity", ErrStorageUnavailable) + } + defer releaseAttachmentSlot(s.attachmentUploadSlots) + + stagedFile, err := os.CreateTemp("", "mtusi-attachment-*") + if err != nil { + return domain.Attachment{}, fmt.Errorf("create bounded upload staging file: %w", err) + } + stagedPath := stagedFile.Name() + defer func() { + _ = stagedFile.Close() + _ = os.Remove(stagedPath) + }() + + hasher := sha256.New() + limitedBody := io.LimitReader(command.Body, domain.MaxAttachmentSizeBytes+1) + sizeBytes, err := io.Copy(io.MultiWriter(stagedFile, hasher), limitedBody) + if err != nil { + return domain.Attachment{}, fmt.Errorf("%w: read attachment body", ErrInvalidInput) + } + if sizeBytes > domain.MaxAttachmentSizeBytes { + return domain.Attachment{}, ErrPayloadTooLarge + } + + if _, err = stagedFile.Seek(0, io.SeekStart); err != nil { + return domain.Attachment{}, fmt.Errorf("rewind staged attachment: %w", err) + } + imageConfig, format, err := image.DecodeConfig(stagedFile) + if err != nil { + return domain.Attachment{}, fmt.Errorf("%w: decode image header", ErrInvalidImage) + } + actualContentType, supported := contentTypeForImageFormat(format) + if !supported || actualContentType != command.DeclaredContentType { + return domain.Attachment{}, ErrUnsupportedMediaType + } + if imageConfig.Width < 1 || imageConfig.Height < 1 { + return domain.Attachment{}, ErrInvalidImage + } + if int64(imageConfig.Width) > int64(domain.MaxAttachmentPixels)/int64(imageConfig.Height) { + return domain.Attachment{}, ErrPayloadTooLarge + } + + sha256Hex := hex.EncodeToString(hasher.Sum(nil)) + if err = domain.ValidateReadyAttachmentMetadata( + command.Filename, + actualContentType, + sizeBytes, + sha256Hex, + imageConfig.Width, + imageConfig.Height, + ); err != nil { + return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidImage, err) + } + + if _, err = stagedFile.Seek(0, io.SeekStart); err != nil { + return domain.Attachment{}, fmt.Errorf("rewind staged attachment for validation: %w", err) + } + if err = s.validateCompleteImage(ctx, stagedFile); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return domain.Attachment{}, fmt.Errorf("%w: wait for image decoder", ErrStorageUnavailable) + } + return domain.Attachment{}, fmt.Errorf("%w: decode complete image", ErrInvalidImage) + } + + now := s.now().UTC() + attachmentID := uuid.NewString() + attachment := domain.Attachment{ + ID: attachmentID, + ChatID: command.ChatID, + OwnerID: command.ActorID, + Filename: command.Filename, + ObjectKey: "attachments/" + command.ChatID + "/" + attachmentID + "/original", + ContentType: actualContentType, + SizeBytes: sizeBytes, + SHA256: sha256Hex, + Width: imageConfig.Width, + Height: imageConfig.Height, + Status: domain.AttachmentStatusPending, + Variants: make([]domain.AttachmentVariant, 0), + CreatedAt: now, + UpdatedAt: now, + } + + attachment, err = s.repository.CreateAttachment(ctx, attachment) + if err != nil { + return domain.Attachment{}, err + } + if _, err = stagedFile.Seek(0, io.SeekStart); err != nil { + s.cleanupFailedAttachment(ctx, attachment, attachmentFinalizeFailedCode, false) + return domain.Attachment{}, fmt.Errorf("rewind staged attachment for storage: %w", err) + } + err = s.objectStore.Put(ctx, ObjectPut{ + Key: attachment.ObjectKey, + ContentType: attachment.ContentType, + SizeBytes: attachment.SizeBytes, + Body: stagedFile, + }) + if err != nil { + s.cleanupFailedAttachment(ctx, attachment, attachmentPutFailedCode, true) + return domain.Attachment{}, fmt.Errorf("%w: put object", ErrStorageUnavailable) + } + + readyAttachment, err := s.repository.MarkAttachmentReady(ctx, attachment.ID, s.now().UTC()) + if err != nil { + // The UPDATE may already have committed even when its response was lost. + // Deleting the object here could turn durable ready metadata into a + // permanently broken attachment. A best-effort read may recover a 201; + // otherwise both sides are preserved for the grace-period reconciler. + if resolved, ok := s.resolveAmbiguousAttachmentFinalize(ctx, attachment); ok { + return resolved, nil + } + return domain.Attachment{}, fmt.Errorf("finalize attachment metadata: %w", err) + } + if readyAttachment.Variants == nil { + readyAttachment.Variants = make([]domain.AttachmentVariant, 0) + } + return readyAttachment, nil +} + +func acquireAttachmentSlot(ctx context.Context, slots chan struct{}) error { + select { + case slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func releaseAttachmentSlot(slots chan struct{}) { + <-slots +} + +func (s *Service) validateCompleteImage(ctx context.Context, body io.Reader) error { + if err := acquireAttachmentSlot(ctx, s.attachmentDecodeSlots); err != nil { + return err + } + defer releaseAttachmentSlot(s.attachmentDecodeSlots) + _, _, err := image.Decode(body) + return err +} + +func (s *Service) resolveAmbiguousAttachmentFinalize( + requestContext context.Context, + attachment domain.Attachment, +) (domain.Attachment, bool) { + resolveContext, cancel := context.WithTimeout( + context.WithoutCancel(requestContext), + s.attachmentCleanupTimeout, + ) + defer cancel() + + stored, err := s.repository.GetAttachment( + resolveContext, + attachment.ID, + attachment.OwnerID, + ) + if err != nil || stored.Status != domain.AttachmentStatusReady { + return domain.Attachment{}, false + } + if stored.Variants == nil { + stored.Variants = make([]domain.AttachmentVariant, 0) + } + return stored, true +} + +func contentTypeForImageFormat(format string) (string, bool) { + switch format { + case "png": + return "image/png", true + case "jpeg": + return "image/jpeg", true + default: + return "", false + } +} + +func (s *Service) cleanupFailedAttachment( + requestContext context.Context, + attachment domain.Attachment, + errorCode string, + deleteObject bool, +) { + metadataContext, metadataCancel := context.WithTimeout( + context.WithoutCancel(requestContext), + s.attachmentCleanupTimeout, + ) + _ = s.repository.MarkAttachmentFailed( + metadataContext, + attachment.ID, + errorCode, + s.now().UTC(), + ) + metadataCancel() + + if !deleteObject { + return + } + deleteContext, deleteCancel := context.WithTimeout( + context.WithoutCancel(requestContext), + s.attachmentCleanupTimeout, + ) + defer deleteCancel() + _ = s.objectStore.Delete(deleteContext, attachment.ObjectKey) +} diff --git a/internal/application/attachments_test.go b/internal/application/attachments_test.go new file mode 100644 index 0000000..926c0a4 --- /dev/null +++ b/internal/application/attachments_test.go @@ -0,0 +1,524 @@ +package application_test + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "strings" + "sync" + "testing" + "time" + + "mtusi2026lab4/internal/adapter/memory" + . "mtusi2026lab4/internal/application" + "mtusi2026lab4/internal/domain" +) + +type fakeObjectStore struct { + mu sync.Mutex + objects map[string][]byte + putError error + blockDelete bool + putCalls int + deleteCalls int + lastPutKey string +} + +type alwaysFailReader struct{} + +func (alwaysFailReader) Read([]byte) (int, error) { + return 0, errors.New("injected interrupted upload") +} + +type finalizeFailureRepository struct { + Repository +} + +func (repository *finalizeFailureRepository) MarkAttachmentReady( + context.Context, + string, + time.Time, +) (domain.Attachment, error) { + return domain.Attachment{}, errors.New("injected metadata finalize failure") +} + +type commitThenDisconnectRepository struct { + Repository + hideReadback bool +} + +func (repository *commitThenDisconnectRepository) MarkAttachmentReady( + ctx context.Context, + attachmentID string, + updatedAt time.Time, +) (domain.Attachment, error) { + if _, err := repository.Repository.MarkAttachmentReady(ctx, attachmentID, updatedAt); err != nil { + return domain.Attachment{}, err + } + return domain.Attachment{}, errors.New("injected connection loss after commit") +} + +func (repository *commitThenDisconnectRepository) GetAttachment( + ctx context.Context, + attachmentID string, + actorID string, +) (domain.Attachment, error) { + if repository.hideReadback { + return domain.Attachment{}, errors.New("injected database outage during readback") + } + return repository.Repository.GetAttachment(ctx, attachmentID, actorID) +} + +func newFakeObjectStore() *fakeObjectStore { + return &fakeObjectStore{objects: make(map[string][]byte)} +} + +func (store *fakeObjectStore) Put(_ context.Context, object ObjectPut) error { + store.mu.Lock() + defer store.mu.Unlock() + store.putCalls++ + store.lastPutKey = object.Key + if store.putError != nil { + return store.putError + } + payload, err := io.ReadAll(object.Body) + if err != nil { + return err + } + if int64(len(payload)) != object.SizeBytes { + return errors.New("declared object size differs") + } + store.objects[object.Key] = payload + return nil +} + +func (store *fakeObjectStore) Get(_ context.Context, key string) (ObjectContent, error) { + store.mu.Lock() + defer store.mu.Unlock() + payload, exists := store.objects[key] + if !exists { + return ObjectContent{}, errors.New("object not found") + } + copyOfPayload := append([]byte(nil), payload...) + return ObjectContent{ + Body: io.NopCloser(bytes.NewReader(copyOfPayload)), + SizeBytes: int64(len(copyOfPayload)), + }, nil +} + +func (store *fakeObjectStore) Delete(ctx context.Context, key string) error { + store.mu.Lock() + store.deleteCalls++ + blockDelete := store.blockDelete + store.mu.Unlock() + if blockDelete { + <-ctx.Done() + return ctx.Err() + } + store.mu.Lock() + defer store.mu.Unlock() + delete(store.objects, key) + return nil +} + +func newAttachmentService( + t *testing.T, + objectStore ObjectStore, +) (*Service, *memory.MemoryStore, string, string) { + t.Helper() + legacyStore := memory.NewMemoryStore() + repository := memory.NewMemoryRepository(legacyStore) + user, err := repository.CreateUser(context.Background(), "uploader", "Uploader", "hash") + if err != nil { + t.Fatal(err) + } + chat, err := repository.CreateChat(context.Background(), "Images", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + codec, err := NewCursorCodec("attachment-test-key-with-more-than-thirty-two-bytes") + if err != nil { + t.Fatal(err) + } + service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore)) + if err != nil { + t.Fatal(err) + } + return service, legacyStore, user.ID, chat.ID +} + +func encodeTestImage(t *testing.T, contentType string) []byte { + t.Helper() + var payload bytes.Buffer + img := image.NewRGBA(image.Rect(0, 0, 2, 2)) + img.Set(0, 0, color.RGBA{R: 255, A: 255}) + var err error + if contentType == "image/png" { + err = png.Encode(&payload, img) + } else { + err = jpeg.Encode(&payload, img, nil) + } + if err != nil { + t.Fatal(err) + } + return payload.Bytes() +} + +func corruptPNGAfterPrefix(t *testing.T) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, 512, 512)) + state := uint32(0x12345678) + for index := range img.Pix { + state ^= state << 13 + state ^= state >> 17 + state ^= state << 5 + img.Pix[index] = byte(state) + } + var payload bytes.Buffer + if err := png.Encode(&payload, img); err != nil { + t.Fatal(err) + } + corrupted := payload.Bytes() + if len(corrupted) <= 70_000 { + t.Fatalf("test PNG is unexpectedly small: %d", len(corrupted)) + } + corrupted[70_000] ^= 0xff + return corrupted +} + +func jpegWithoutScanData(t *testing.T) []byte { + t.Helper() + payload := encodeTestImage(t, "image/jpeg") + sos := bytes.Index(payload, []byte{0xff, 0xda}) + if sos < 0 || sos+4 > len(payload) { + t.Fatal("encoded JPEG has no SOS marker") + } + segmentLength := int(binary.BigEndian.Uint16(payload[sos+2 : sos+4])) + endOfHeader := sos + 2 + segmentLength + if endOfHeader > len(payload) { + t.Fatal("invalid SOS segment length in test JPEG") + } + return append([]byte(nil), payload[:endOfHeader]...) +} + +func TestUploadAttachmentStoresValidatedImage(t *testing.T) { + objectStore := newFakeObjectStore() + service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore) + payload := encodeTestImage(t, "image/png") + + attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, + ChatID: chatID, + Filename: "кот.png", + DeclaredContentType: "image/png", + Body: bytes.NewReader(payload), + }) + if err != nil { + t.Fatal(err) + } + if attachment.Status != domain.AttachmentStatusReady { + t.Fatalf("status = %q, want ready", attachment.Status) + } + if attachment.ObjectKey == "" || strings.Contains(attachment.ObjectKey, attachment.Filename) { + t.Fatalf("object key must be server-generated: %q", attachment.ObjectKey) + } + if attachment.Width != 2 || attachment.Height != 2 || len(attachment.SHA256) != 64 { + t.Fatalf("unexpected image metadata: %+v", attachment) + } + if !bytes.Equal(objectStore.objects[attachment.ObjectKey], payload) { + t.Fatal("stored object differs from uploaded bytes") + } + stored, err := metadataStore.GetAttachmentByID(attachment.ID) + if err != nil || stored.Status != domain.AttachmentStatusReady { + t.Fatalf("ready metadata was not persisted: %+v, %v", stored, err) + } +} + +func TestUploadAttachmentRejectsInvalidInputBeforeStorage(t *testing.T) { + testCases := []struct { + name string + filename string + declaredType string + payload []byte + wantError error + }{ + {name: "unsafe filename", filename: "../image.png", declaredType: "image/png", payload: encodeTestImage(t, "image/png"), wantError: ErrInvalidInput}, + {name: "unsupported declared type", filename: "image.gif", declaredType: "image/gif", payload: []byte("GIF89a"), wantError: ErrUnsupportedMediaType}, + {name: "MIME substitution", filename: "image.jpg", declaredType: "image/jpeg", payload: encodeTestImage(t, "image/png"), wantError: ErrUnsupportedMediaType}, + {name: "corrupt image", filename: "image.png", declaredType: "image/png", payload: []byte("not a png"), wantError: ErrInvalidImage}, + {name: "PNG header without image data", filename: "truncated.png", declaredType: "image/png", payload: pngConfigurationOnly(1, 1), wantError: ErrInvalidImage}, + {name: "PNG corruption after 64 KiB", filename: "corrupted.png", declaredType: "image/png", payload: corruptPNGAfterPrefix(t), wantError: ErrInvalidImage}, + {name: "JPEG without scan data", filename: "truncated.jpg", declaredType: "image/jpeg", payload: jpegWithoutScanData(t), wantError: ErrInvalidImage}, + {name: "too large", filename: "image.png", declaredType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantError: ErrPayloadTooLarge}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + objectStore := newFakeObjectStore() + service, _, actorID, chatID := newAttachmentService(t, objectStore) + _, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, + ChatID: chatID, + Filename: testCase.filename, + DeclaredContentType: testCase.declaredType, + Body: bytes.NewReader(testCase.payload), + }) + if !errors.Is(err, testCase.wantError) { + t.Fatalf("error = %v, want %v", err, testCase.wantError) + } + if objectStore.putCalls != 0 { + t.Fatalf("invalid upload reached object storage %d times", objectStore.putCalls) + } + }) + } +} + +func TestUploadAttachmentCompensatesObjectStoreFailure(t *testing.T) { + objectStore := newFakeObjectStore() + objectStore.putError = errors.New("injected put failure") + service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore) + + _, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, + ChatID: chatID, + Filename: "image.jpg", + DeclaredContentType: "image/jpeg", + Body: bytes.NewReader(encodeTestImage(t, "image/jpeg")), + }) + if !errors.Is(err, ErrStorageUnavailable) { + t.Fatalf("error = %v, want ErrStorageUnavailable", err) + } + if objectStore.deleteCalls != 1 { + t.Fatalf("compensating deletes = %d, want 1", objectStore.deleteCalls) + } + + objectStore.mu.Lock() + keyParts := strings.Split(objectStore.lastPutKey, "/") + objectStore.mu.Unlock() + if len(keyParts) != 4 { + t.Fatalf("unexpected generated object key: %q", objectStore.lastPutKey) + } + failedAttachment, lookupErr := metadataStore.GetAttachmentByID(keyParts[2]) + if lookupErr != nil { + t.Fatal(lookupErr) + } + if failedAttachment.Status != domain.AttachmentStatusFailed || failedAttachment.ErrorCode == nil { + t.Fatalf("failed upload metadata was not finalized: %+v", failedAttachment) + } +} + +func TestUploadAttachmentMarksFailedBeforeBlockedObjectCleanup(t *testing.T) { + objectStore := newFakeObjectStore() + objectStore.putError = errors.New("injected put failure") + objectStore.blockDelete = true + service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore) + + startedAt := time.Now() + _, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, ChatID: chatID, Filename: "blocked-cleanup.png", + DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")), + }) + if !errors.Is(err, ErrStorageUnavailable) { + t.Fatalf("error = %v, want ErrStorageUnavailable", err) + } + if elapsed := time.Since(startedAt); elapsed > 2*time.Second { + t.Fatalf("bounded cleanup took %s", elapsed) + } + keyParts := strings.Split(objectStore.lastPutKey, "/") + failed, lookupErr := metadataStore.GetAttachmentByID(keyParts[2]) + if lookupErr != nil { + t.Fatal(lookupErr) + } + if failed.Status != domain.AttachmentStatusFailed || failed.ErrorCode == nil { + t.Fatalf("blocked object cleanup consumed DB cleanup context: %+v", failed) + } +} + +func TestUploadAttachmentRejectsInterruptedBodyBeforeMetadata(t *testing.T) { + objectStore := newFakeObjectStore() + service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore) + payload := encodeTestImage(t, "image/png") + + _, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, + ChatID: chatID, + Filename: "interrupted.png", + DeclaredContentType: "image/png", + Body: io.MultiReader(bytes.NewReader(payload[:8]), alwaysFailReader{}), + }) + if !errors.Is(err, ErrInvalidInput) { + t.Fatalf("error = %v, want ErrInvalidInput", err) + } + if objectStore.putCalls != 0 { + t.Fatal("interrupted upload reached object storage") + } + if _, lookupErr := metadataStore.GetAttachmentByID("missing"); !errors.Is(lookupErr, domain.ErrNotFound) { + t.Fatalf("unexpected metadata lookup result: %v", lookupErr) + } +} + +func TestUploadAttachmentPreservesObjectOnMetadataFinalizeFailure(t *testing.T) { + legacyStore := memory.NewMemoryStore() + repository := memory.NewMemoryRepository(legacyStore) + user, err := repository.CreateUser(context.Background(), "finalize", "Finalize", "hash") + if err != nil { + t.Fatal(err) + } + chat, err := repository.CreateChat(context.Background(), "Finalize", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + codec, err := NewCursorCodec("finalize-test-key-with-more-than-thirty-two-bytes") + if err != nil { + t.Fatal(err) + } + objectStore := newFakeObjectStore() + service, err := NewService( + &finalizeFailureRepository{Repository: repository}, + codec, + time.Hour, + WithObjectStore(objectStore), + ) + if err != nil { + t.Fatal(err) + } + + _, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: user.ID, + ChatID: chat.ID, + Filename: "finalize.png", + DeclaredContentType: "image/png", + Body: bytes.NewReader(encodeTestImage(t, "image/png")), + }) + if err == nil { + t.Fatal("metadata finalize failure was hidden") + } + if objectStore.deleteCalls != 0 || len(objectStore.objects) != 1 { + t.Fatalf("ambiguous object must be preserved: deletes=%d objects=%d", objectStore.deleteCalls, len(objectStore.objects)) + } + keyParts := strings.Split(objectStore.lastPutKey, "/") + pending, lookupErr := legacyStore.GetAttachmentByID(keyParts[2]) + if lookupErr != nil { + t.Fatal(lookupErr) + } + if pending.Status != domain.AttachmentStatusPending || pending.ErrorCode != nil { + t.Fatalf("ambiguous metadata must remain reconcilable: %+v", pending) + } +} + +func TestUploadAttachmentRecoversWhenCommitResponseIsLost(t *testing.T) { + legacyStore := memory.NewMemoryStore() + baseRepository := memory.NewMemoryRepository(legacyStore) + user, err := baseRepository.CreateUser(context.Background(), "commit-lost", "Commit Lost", "hash") + if err != nil { + t.Fatal(err) + } + chat, err := baseRepository.CreateChat(context.Background(), "Commit Lost", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + codec, err := NewCursorCodec("commit-lost-test-key-with-more-than-thirty-two-bytes") + if err != nil { + t.Fatal(err) + } + objectStore := newFakeObjectStore() + repository := &commitThenDisconnectRepository{Repository: baseRepository} + service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore)) + if err != nil { + t.Fatal(err) + } + + attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png", + DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")), + }) + if err != nil { + t.Fatalf("committed ready attachment was not recovered: %v", err) + } + if attachment.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 { + t.Fatalf("lost commit response damaged attachment: attachment=%+v deletes=%d objects=%d", + attachment, objectStore.deleteCalls, len(objectStore.objects)) + } +} + +func TestUploadAttachmentNeverDeletesObjectAfterAmbiguousCommit(t *testing.T) { + legacyStore := memory.NewMemoryStore() + baseRepository := memory.NewMemoryRepository(legacyStore) + user, err := baseRepository.CreateUser(context.Background(), "commit-outage", "Commit Outage", "hash") + if err != nil { + t.Fatal(err) + } + chat, err := baseRepository.CreateChat(context.Background(), "Commit Outage", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + codec, err := NewCursorCodec("commit-outage-test-key-with-more-than-thirty-two-bytes") + if err != nil { + t.Fatal(err) + } + objectStore := newFakeObjectStore() + repository := &commitThenDisconnectRepository{Repository: baseRepository, hideReadback: true} + service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore)) + if err != nil { + t.Fatal(err) + } + + _, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png", + DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")), + }) + if err == nil { + t.Fatal("database outage after commit was hidden") + } + keyParts := strings.Split(objectStore.lastPutKey, "/") + stored, lookupErr := legacyStore.GetAttachmentByID(keyParts[2]) + if lookupErr != nil { + t.Fatal(lookupErr) + } + if stored.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 { + t.Fatalf("committed ready metadata lost its original: metadata=%+v deletes=%d objects=%d", + stored, objectStore.deleteCalls, len(objectStore.objects)) + } +} + +func TestUploadAttachmentRejectsPixelLimitBeforeFullDecode(t *testing.T) { + objectStore := newFakeObjectStore() + service, _, actorID, chatID := newAttachmentService(t, objectStore) + + _, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{ + ActorID: actorID, + ChatID: chatID, + Filename: "huge.png", + DeclaredContentType: "image/png", + Body: bytes.NewReader(pngConfigurationOnly(10_000, 4_001)), + }) + if !errors.Is(err, ErrPayloadTooLarge) { + t.Fatalf("error = %v, want ErrPayloadTooLarge", err) + } + if objectStore.putCalls != 0 { + t.Fatalf("oversized decoded image reached object storage %d times", objectStore.putCalls) + } +} + +func pngConfigurationOnly(width uint32, height uint32) []byte { + payload := make([]byte, 33) + copy(payload[0:8], []byte{137, 80, 78, 71, 13, 10, 26, 10}) + binary.BigEndian.PutUint32(payload[8:12], 13) + copy(payload[12:16], "IHDR") + binary.BigEndian.PutUint32(payload[16:20], width) + binary.BigEndian.PutUint32(payload[20:24], height) + payload[24] = 8 + payload[25] = 6 + payload[26] = 0 + payload[27] = 0 + payload[28] = 0 + binary.BigEndian.PutUint32(payload[29:33], crc32.ChecksumIEEE(payload[12:29])) + return payload +} diff --git a/internal/application/commands.go b/internal/application/commands.go index e63e11e..c33b5c1 100644 --- a/internal/application/commands.go +++ b/internal/application/commands.go @@ -1,5 +1,7 @@ package application +import "io" + type CreateUserCommand struct { Username string DisplayName string @@ -21,7 +23,16 @@ type SendMessageCommand struct { ActorID string ChatID string Text string + AttachmentIDs []string Method string CanonicalPath string IdempotencyKey string } + +type UploadAttachmentCommand struct { + ActorID string + ChatID string + Filename string + DeclaredContentType string + Body io.Reader +} diff --git a/internal/application/cursor_test.go b/internal/application/cursor_test.go index 91e848d..c2be0b9 100644 --- a/internal/application/cursor_test.go +++ b/internal/application/cursor_test.go @@ -1,7 +1,9 @@ package application import ( + "encoding/base64" "errors" + "strings" "testing" "time" @@ -47,11 +49,13 @@ func TestCursorCodecRejectsTamperingAndWrongScope(t *testing.T) { t.Fatal(err) } - replacement := "A" - if token[len(token)-1:] == replacement { - replacement = "B" + parts := strings.Split(token, ".") + signature, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal(err) } - tampered := token[:len(token)-1] + replacement + signature[0] ^= 1 + tampered := parts[0] + "." + base64.RawURLEncoding.EncodeToString(signature) if _, err = codec.DecodeChat(tampered, "9fa3f49c-e62e-4209-8414-d6d5f69805bf"); !errors.Is(err, ErrInvalidCursor) { t.Fatalf("tampered cursor: got %v, want ErrInvalidCursor", err) } @@ -68,3 +72,25 @@ func TestCursorCodecRequiresStrongSharedKey(t *testing.T) { t.Fatal("expected short cursor signing key to be rejected") } } + +func TestCursorCodecRejectsSignedInvalidFieldTypesBeforeSQL(t *testing.T) { + codec, err := NewCursorCodec(cursorTestKey) + if err != nil { + t.Fatal(err) + } + chatID := "9fa3f49c-e62e-4209-8414-d6d5f69805bf" + validMessageID := "42eea32e-d9b5-4bc6-b4fc-94e9b0477adf" + testCases := []cursorPayload{ + {Version: cursorVersion, Kind: "message", CreatedAt: "not-a-time", ID: validMessageID, ScopeID: chatID}, + {Version: cursorVersion, Kind: "message", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), ID: "not-a-uuid", ScopeID: chatID}, + } + for _, payload := range testCases { + token, encodeErr := codec.encode(payload) + if encodeErr != nil { + t.Fatal(encodeErr) + } + if _, decodeErr := codec.DecodeMessage(token, chatID); !errors.Is(decodeErr, ErrInvalidCursor) { + t.Fatalf("invalid signed cursor returned %v, want ErrInvalidCursor", decodeErr) + } + } +} diff --git a/internal/application/errors.go b/internal/application/errors.go index 9009c36..336f54b 100644 --- a/internal/application/errors.go +++ b/internal/application/errors.go @@ -3,9 +3,14 @@ package application import "errors" var ( - ErrInvalidCursor = errors.New("invalid cursor") - ErrInvalidCredentials = errors.New("invalid credentials") - ErrInvalidInput = errors.New("invalid input") - ErrCredentialState = errors.New("invalid stored credential state") - ErrIdempotencyConflict = errors.New("idempotency key is already used for another request") + ErrInvalidCursor = errors.New("invalid cursor") + ErrInvalidCredentials = errors.New("invalid credentials") + ErrInvalidInput = errors.New("invalid input") + ErrCredentialState = errors.New("invalid stored credential state") + ErrIdempotencyConflict = errors.New("idempotency key is already used for another request") + ErrAttachmentNotReady = errors.New("attachment is not ready") + ErrPayloadTooLarge = errors.New("payload too large") + ErrUnsupportedMediaType = errors.New("unsupported media type") + ErrInvalidImage = errors.New("invalid image") + ErrStorageUnavailable = errors.New("object storage unavailable") ) diff --git a/internal/application/messages.go b/internal/application/messages.go index 9225f03..4003b06 100644 --- a/internal/application/messages.go +++ b/internal/application/messages.go @@ -7,6 +7,8 @@ import ( "fmt" "mtusi2026lab4/internal/domain" + + "github.com/google/uuid" ) type canonicalMessageRequest struct { @@ -15,13 +17,21 @@ type canonicalMessageRequest struct { } func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) (domain.Message, error) { - if err := domain.ValidateMessageText(command.Text); err != nil { + attachmentIDs := append([]string(nil), command.AttachmentIDs...) + if attachmentIDs == nil { + attachmentIDs = make([]string, 0) + } + + if err := domain.ValidateMessageInput(command.Text, attachmentIDs); err != nil { return domain.Message{}, fmt.Errorf("%w: %v", ErrInvalidInput, err) } + for index, attachmentID := range attachmentIDs { + attachmentIDs[index] = uuid.MustParse(attachmentID).String() + } canonicalBody, err := json.Marshal(canonicalMessageRequest{ Text: command.Text, - AttachmentIDs: make([]string, 0), + AttachmentIDs: attachmentIDs, }) if err != nil { return domain.Message{}, fmt.Errorf("canonicalize message request: %w", err) @@ -31,6 +41,7 @@ func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) ( ActorID: command.ActorID, ChatID: command.ChatID, Text: command.Text, + AttachmentIDs: attachmentIDs, Method: command.Method, CanonicalPath: command.CanonicalPath, Key: command.IdempotencyKey, diff --git a/internal/application/ports.go b/internal/application/ports.go index 442ad5c..e1ac839 100644 --- a/internal/application/ports.go +++ b/internal/application/ports.go @@ -2,6 +2,7 @@ package application import ( "context" + "io" "time" "mtusi2026lab4/internal/domain" @@ -28,6 +29,7 @@ type IdempotentMessageWrite struct { ActorID string ChatID string Text string + AttachmentIDs []string Method string CanonicalPath string Key string @@ -40,9 +42,35 @@ type MessageRepository interface { GetMessages(ctx context.Context, chatID, callerID string, limit int, cursor *domain.MessageCursor) ([]domain.Message, bool, error) } +type AttachmentRepository interface { + CreateAttachment(ctx context.Context, attachment domain.Attachment) (domain.Attachment, error) + MarkAttachmentReady(ctx context.Context, attachmentID string, updatedAt time.Time) (domain.Attachment, error) + MarkAttachmentFailed(ctx context.Context, attachmentID, errorCode string, updatedAt time.Time) error + GetAttachment(ctx context.Context, attachmentID, callerID string) (domain.Attachment, error) +} + type Repository interface { UserRepository SessionRepository ChatRepository MessageRepository + AttachmentRepository +} + +type ObjectPut struct { + Key string + ContentType string + SizeBytes int64 + Body io.Reader +} + +type ObjectContent struct { + Body io.ReadCloser + SizeBytes int64 +} + +type ObjectStore interface { + Put(ctx context.Context, object ObjectPut) error + Get(ctx context.Context, key string) (ObjectContent, error) + Delete(ctx context.Context, key string) error } diff --git a/internal/application/service.go b/internal/application/service.go index f66ea99..4ca76e7 100644 --- a/internal/application/service.go +++ b/internal/application/service.go @@ -6,15 +6,22 @@ import ( ) const MinimumIdempotencyTTL = 24 * time.Hour +const defaultAttachmentCleanupTimeout = time.Second +const defaultAttachmentUploadConcurrency = 4 +const defaultAttachmentDecodeConcurrency = 1 type ServiceOption func(*Service) error type Service struct { - repository Repository - cursors *CursorCodec - sessionTTL time.Duration - idempotencyTTL time.Duration - now func() time.Time + repository Repository + cursors *CursorCodec + sessionTTL time.Duration + idempotencyTTL time.Duration + now func() time.Time + objectStore ObjectStore + attachmentCleanupTimeout time.Duration + attachmentUploadSlots chan struct{} + attachmentDecodeSlots chan struct{} } func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Duration, options ...ServiceOption) (*Service, error) { @@ -28,11 +35,14 @@ func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Dur return nil, fmt.Errorf("session TTL must be positive") } service := &Service{ - repository: repository, - cursors: cursors, - sessionTTL: sessionTTL, - idempotencyTTL: MinimumIdempotencyTTL, - now: func() time.Time { return time.Now().UTC() }, + repository: repository, + cursors: cursors, + sessionTTL: sessionTTL, + idempotencyTTL: MinimumIdempotencyTTL, + now: func() time.Time { return time.Now().UTC() }, + attachmentCleanupTimeout: defaultAttachmentCleanupTimeout, + attachmentUploadSlots: make(chan struct{}, defaultAttachmentUploadConcurrency), + attachmentDecodeSlots: make(chan struct{}, defaultAttachmentDecodeConcurrency), } for _, option := range options { if err := option(service); err != nil { @@ -51,3 +61,13 @@ func WithIdempotencyTTL(ttl time.Duration) ServiceOption { return nil } } + +func WithObjectStore(objectStore ObjectStore) ServiceOption { + return func(service *Service) error { + if objectStore == nil { + return fmt.Errorf("object store is required") + } + service.objectStore = objectStore + return nil + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 01459b9..6512a49 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,6 +35,13 @@ type Config struct { RedisOperationTimeout time.Duration RedisPoolSize int + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKeyID string + S3SecretAccessKey string + S3OperationTimeout time.Duration + ReadinessTimeout time.Duration ShutdownDrainDelay time.Duration ShutdownTimeout time.Duration @@ -82,6 +89,24 @@ func Load() (Config, error) { return Config{}, err } + s3AccessKeyID, err := requiredValueOrFile( + "S3_ACCESS_KEY_ID", + "S3_ACCESS_KEY_ID_FILE", + ) + if err != nil { + return Config{}, err + } + s3SecretAccessKey, err := requiredValueOrFile( + "S3_SECRET_ACCESS_KEY", + "S3_SECRET_ACCESS_KEY_FILE", + ) + if err != nil { + return Config{}, err + } + s3OperationTimeout, err := durationFromEnv("S3_OPERATION_TIMEOUT", 5*time.Second) + if err != nil { + return Config{}, err + } databaseMaxConns, err := int32FromEnv("DB_MAX_CONNS", 10, 1) if err != nil { return Config{}, err @@ -223,6 +248,12 @@ func Load() (Config, error) { RedisOperationTimeout: redisOperationTimeout, RedisPoolSize: int(redisPoolSize), + S3Endpoint: envOrDefault("S3_ENDPOINT", "http://minio:9000"), + S3Region: envOrDefault("S3_REGION", "us-east-1"), + S3Bucket: envOrDefault("S3_BUCKET", "messenger-attachments"), + S3AccessKeyID: s3AccessKeyID, + S3SecretAccessKey: s3SecretAccessKey, + S3OperationTimeout: s3OperationTimeout, ReadinessTimeout: readinessTimeout, ShutdownDrainDelay: shutdownDrainDelay, ShutdownTimeout: shutdownTimeout, diff --git a/internal/domain/attachment.go b/internal/domain/attachment.go new file mode 100644 index 0000000..894c6e8 --- /dev/null +++ b/internal/domain/attachment.go @@ -0,0 +1,119 @@ +package domain + +import ( + "encoding/hex" + "fmt" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const ( + MaxAttachmentFilenameCodePoints = 255 + MaxAttachmentSizeBytes = 10 * 1024 * 1024 + MaxAttachmentPixels = 40_000_000 +) + +type AttachmentVariant struct { + Name string + ContentType string + SizeBytes int64 + Width int + Height int + SHA256 string +} + +type AttachmentStatus string + +const ( + AttachmentStatusPending AttachmentStatus = "pending" + AttachmentStatusReady AttachmentStatus = "ready" + AttachmentStatusFailed AttachmentStatus = "failed" +) + +// Attachment contains metadata only. ObjectKey points to bytes in the +// private object store and must never be derived from Filename. +type Attachment struct { + ID string + ChatID string + OwnerID string + Filename string + ObjectKey string + ContentType string + SizeBytes int64 + SHA256 string + Width int + Height int + Status AttachmentStatus + Variants []AttachmentVariant + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time +} + +func ValidateAttachmentFilename(filename string) error { + if !utf8.ValidString(filename) { + return fmt.Errorf("filename must be valid UTF-8") + } + filenameLength := utf8.RuneCountInString(filename) + if filenameLength < 1 || filenameLength > MaxAttachmentFilenameCodePoints { + return fmt.Errorf("filename length must be between 1 and %d code points", MaxAttachmentFilenameCodePoints) + } + if strings.TrimSpace(filename) == "" || filename == "." || filename == ".." { + return fmt.Errorf("filename is unsafe") + } + if strings.ContainsAny(filename, `/\\`) { + return fmt.Errorf("filename must not contain path separators") + } + for _, character := range filename { + if unicode.IsControl(character) { + return fmt.Errorf("filename must not contain control characters") + } + } + return nil +} + +func ValidateAttachmentContentType(contentType string) error { + if contentType != "image/png" && contentType != "image/jpeg" { + return fmt.Errorf("content type must be image/png or image/jpeg") + } + return nil +} + +// ValidateReadyAttachmentMetadata checks the bounded metadata produced after +// the upload stream has been identified, decoded and hashed. It deliberately +// does not trust an HTTP Content-Type header. +func ValidateReadyAttachmentMetadata( + filename string, + contentType string, + sizeBytes int64, + sha256Hex string, + width int, + height int, +) error { + if err := ValidateAttachmentFilename(filename); err != nil { + return err + } + if err := ValidateAttachmentContentType(contentType); err != nil { + return err + } + if sizeBytes < 1 || sizeBytes > MaxAttachmentSizeBytes { + return fmt.Errorf("attachment size must be between 1 and %d bytes", MaxAttachmentSizeBytes) + } + if width < 1 || height < 1 { + return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels) + } + if int64(width) > int64(MaxAttachmentPixels)/int64(height) { + return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels) + } + if len(sha256Hex) != 64 || sha256Hex != strings.ToLower(sha256Hex) { + return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters") + } + decodedHash, err := hex.DecodeString(sha256Hex) + if err != nil || len(decodedHash) != 32 { + return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters") + } + + return nil +} diff --git a/internal/domain/attachment_test.go b/internal/domain/attachment_test.go new file mode 100644 index 0000000..ec7c101 --- /dev/null +++ b/internal/domain/attachment_test.go @@ -0,0 +1,59 @@ +package domain + +import ( + "math" + "strings" + "testing" +) + +func TestValidateReadyAttachmentMetadata(t *testing.T) { + validHash := strings.Repeat("ab", 32) + + testCases := []struct { + name string + filename string + contentType string + sizeBytes int64 + sha256 string + width int + height int + wantError bool + }{ + {name: "png", filename: "кот.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1}, + {name: "jpeg at limits", filename: strings.Repeat("界", 255), contentType: "image/jpeg", sizeBytes: MaxAttachmentSizeBytes, sha256: validHash, width: 8_000, height: 5_000}, + {name: "empty filename", filename: "", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "blank filename", filename: " ", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "dot segment", filename: "..", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "forward slash", filename: "folder/image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "backslash", filename: `folder\\image.png`, contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "control character", filename: "image\n.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "C1 control character", filename: "image\u0085.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "invalid UTF-8", filename: string([]byte{'x', 0xff, '.', 'p', 'n', 'g'}), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "filename too long", filename: strings.Repeat("a", 256), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "unsupported MIME", filename: "image.gif", contentType: "image/gif", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "empty file", filename: "image.png", contentType: "image/png", sizeBytes: 0, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "file too large", filename: "image.png", contentType: "image/png", sizeBytes: MaxAttachmentSizeBytes + 1, sha256: validHash, width: 1, height: 1, wantError: true}, + {name: "zero width", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 0, height: 1, wantError: true}, + {name: "too many pixels", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 8_001, height: 5_000, wantError: true}, + {name: "dimension multiplication cannot overflow", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: math.MaxInt, height: math.MaxInt, wantError: true}, + {name: "short hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: "ab", width: 1, height: 1, wantError: true}, + {name: "uppercase hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.ToUpper(validHash), width: 1, height: 1, wantError: true}, + {name: "non-hex hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.Repeat("zz", 32), width: 1, height: 1, wantError: true}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + err := ValidateReadyAttachmentMetadata( + testCase.filename, + testCase.contentType, + testCase.sizeBytes, + testCase.sha256, + testCase.width, + testCase.height, + ) + if (err != nil) != testCase.wantError { + t.Fatalf("ValidateReadyAttachmentMetadata() error = %v, wantError %v", err, testCase.wantError) + } + }) + } +} diff --git a/internal/domain/message.go b/internal/domain/message.go index ec4fe47..b661222 100644 --- a/internal/domain/message.go +++ b/internal/domain/message.go @@ -3,11 +3,12 @@ package domain import "time" type Message struct { - ID string - ChatID string - SenderID string - Text string - CreatedAt time.Time + ID string + ChatID string + SenderID string + Text string + AttachmentIDs []string + CreatedAt time.Time } type MessageCursor struct { diff --git a/internal/domain/message_test.go b/internal/domain/message_test.go new file mode 100644 index 0000000..3516a5e --- /dev/null +++ b/internal/domain/message_test.go @@ -0,0 +1,70 @@ +package domain + +import ( + "strings" + "testing" +) + +func TestValidateMessageInput(t *testing.T) { + validAttachment := "11111111-1111-4111-8111-111111111111" + + testCases := []struct { + name string + text string + attachmentIDs []string + wantError bool + }{ + {name: "text only", text: "hello"}, + {name: "attachment only", attachmentIDs: []string{validAttachment}}, + {name: "whitespace with attachment", text: " ", attachmentIDs: []string{validAttachment}}, + {name: "empty", wantError: true}, + {name: "whitespace only", text: " ", wantError: true}, + {name: "text too long", text: strings.Repeat("界", 4001), wantError: true}, + {name: "invalid attachment uuid", text: "hello", attachmentIDs: []string{"not-a-uuid"}, wantError: true}, + { + name: "duplicate attachment uuid ignores case", + text: "hello", + attachmentIDs: []string{ + "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA", + "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + }, + wantError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + err := ValidateMessageInput(testCase.text, testCase.attachmentIDs) + if (err != nil) != testCase.wantError { + t.Fatalf("ValidateMessageInput() error = %v, wantError %v", err, testCase.wantError) + } + }) + } +} + +func TestValidateMessageInputAttachmentLimit(t *testing.T) { + attachmentIDs := []string{ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002", + "00000000-0000-4000-8000-000000000003", + "00000000-0000-4000-8000-000000000004", + "00000000-0000-4000-8000-000000000005", + "00000000-0000-4000-8000-000000000006", + "00000000-0000-4000-8000-000000000007", + "00000000-0000-4000-8000-000000000008", + "00000000-0000-4000-8000-000000000009", + "00000000-0000-4000-8000-000000000010", + } + + if err := ValidateMessageInput("", attachmentIDs); err != nil { + t.Fatalf("10 attachments must be accepted: %v", err) + } + + attachmentIDs = append( + attachmentIDs, + "00000000-0000-4000-8000-000000000011", + ) + if err := ValidateMessageInput("hello", attachmentIDs); err == nil { + t.Fatal("11 attachments must be rejected") + } +} diff --git a/internal/domain/validation.go b/internal/domain/validation.go index 6ec3b33..5d00b65 100644 --- a/internal/domain/validation.go +++ b/internal/domain/validation.go @@ -9,6 +9,8 @@ import ( "github.com/google/uuid" ) +const MaxMessageAttachments = 10 + var usernameRegex = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) func ValidateUser(username, displayName string) error { @@ -65,13 +67,35 @@ func ValidateChatInput(title string, memberIDs []string, callerID string) error return nil } -func ValidateMessageText(text string) error { +func ValidateMessageInput(text string, attachmentIDs []string) error { textRunes := utf8.RuneCountInString(text) - if textRunes < 1 || textRunes > 4000 { - return fmt.Errorf("message text length must be between 1 and 4000 code points") + if textRunes > 4000 { + return fmt.Errorf("message text length must be at most 4000 code points") } - if strings.TrimSpace(text) == "" { - return fmt.Errorf("message text cannot be only whitespace") + if len(attachmentIDs) > MaxMessageAttachments { + return fmt.Errorf("attachment_ids must contain at most %d items", MaxMessageAttachments) + } + + seen := make(map[string]struct{}, len(attachmentIDs)) + for _, attachmentID := range attachmentIDs { + parsedID, err := uuid.Parse(attachmentID) + if err != nil { + return fmt.Errorf("invalid attachment uuid: %s", attachmentID) + } + canonicalID := parsedID.String() + if _, exists := seen[canonicalID]; exists { + return fmt.Errorf("duplicate attachment_id: %s", attachmentID) + } + seen[canonicalID] = struct{}{} + } + + if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 { + return fmt.Errorf("message must contain non-blank text or at least one attachment") } return nil } + +// ValidateMessageText remains for callers that create text-only messages. +func ValidateMessageText(text string) error { + return ValidateMessageInput(text, nil) +} diff --git a/internal/reconcile/reconciler.go b/internal/reconcile/reconciler.go new file mode 100644 index 0000000..5e72ae7 --- /dev/null +++ b/internal/reconcile/reconciler.go @@ -0,0 +1,229 @@ +package reconcile + +import ( + "context" + "fmt" + "sort" + "time" + + "mtusi2026lab4/internal/adapter/objectstore" + "mtusi2026lab4/internal/domain" +) + +const ( + IssueOrphanObject = "orphan_object" + IssueMissingObject = "missing_object" + IssueStalePending = "stale_pending" + IssueFailedObject = "failed_object" + IssueReferencedMissing = "referenced_missing_object" +) + +type AttachmentRecord struct { + ID string + ObjectKey string + Status domain.AttachmentStatus + UpdatedAt time.Time + Referenced bool +} + +type MetadataStore interface { + ListAttachmentInventory(ctx context.Context) ([]AttachmentRecord, error) + MarkAttachmentFailedIfUnreferenced( + ctx context.Context, + attachmentID string, + expectedStatus domain.AttachmentStatus, + updatedBefore time.Time, + errorCode string, + updatedAt time.Time, + ) (bool, error) +} + +type ObjectInventory interface { + List(ctx context.Context, prefix string) ([]objectstore.StoredObject, error) + Delete(ctx context.Context, key string) error +} + +type Issue struct { + Kind string `json:"kind"` + AttachmentID string `json:"attachment_id,omitempty"` + ObjectKey string `json:"object_key"` + Protected bool `json:"protected,omitempty"` + Action string `json:"action"` +} + +type Result struct { + DryRun bool `json:"dry_run"` + GracePeriod time.Duration `json:"-"` + GracePeriodSeconds int64 `json:"grace_period_seconds"` + ScannedAttachments int `json:"scanned_attachments"` + ScannedObjects int `json:"scanned_objects"` + Issues []Issue `json:"issues"` + DeletedObjects int `json:"deleted_objects"` + FailedAttachments int `json:"failed_attachments"` + Duration time.Duration `json:"-"` + DurationMillis int64 `json:"duration_ms"` +} + +type Reconciler struct { + metadata MetadataStore + objects ObjectInventory + now func() time.Time +} + +func New(metadata MetadataStore, objects ObjectInventory) (*Reconciler, error) { + if metadata == nil || objects == nil { + return nil, fmt.Errorf("metadata and object stores are required") + } + return &Reconciler{ + metadata: metadata, + objects: objects, + now: func() time.Time { return time.Now().UTC() }, + }, nil +} + +func (r *Reconciler) Run( + ctx context.Context, + gracePeriod time.Duration, + apply bool, +) (Result, error) { + if gracePeriod <= 0 { + return Result{}, fmt.Errorf("grace period must be positive") + } + startedAt := r.now().UTC() + cutoff := startedAt.Add(-gracePeriod) + result := Result{ + DryRun: !apply, + GracePeriod: gracePeriod, + GracePeriodSeconds: int64(gracePeriod / time.Second), + Issues: make([]Issue, 0), + } + + attachments, err := r.metadata.ListAttachmentInventory(ctx) + if err != nil { + return Result{}, fmt.Errorf("list attachment metadata: %w", err) + } + objects, err := r.objects.List(ctx, "attachments/") + if err != nil { + return Result{}, fmt.Errorf("list attachment objects: %w", err) + } + result.ScannedAttachments = len(attachments) + result.ScannedObjects = len(objects) + + attachmentsByKey := make(map[string]AttachmentRecord, len(attachments)) + for _, attachment := range attachments { + attachmentsByKey[attachment.ObjectKey] = attachment + } + objectsByKey := make(map[string]objectstore.StoredObject, len(objects)) + for _, object := range objects { + objectsByKey[object.Key] = object + if _, exists := attachmentsByKey[object.Key]; exists { + continue + } + if object.LastModified.IsZero() || object.LastModified.After(cutoff) { + continue + } + issue := Issue{Kind: IssueOrphanObject, ObjectKey: object.Key, Action: "would_delete"} + if apply { + if err = r.objects.Delete(ctx, object.Key); err != nil { + return result, fmt.Errorf("delete orphan object %q: %w", object.Key, err) + } + issue.Action = "deleted" + result.DeletedObjects++ + } + result.Issues = append(result.Issues, issue) + } + + for _, attachment := range attachments { + if attachment.UpdatedAt.After(cutoff) { + continue + } + _, objectExists := objectsByKey[attachment.ObjectKey] + switch { + case attachment.Status == domain.AttachmentStatusReady && !objectExists: + issue := Issue{ + Kind: IssueMissingObject, + AttachmentID: attachment.ID, + ObjectKey: attachment.ObjectKey, + Protected: attachment.Referenced, + Action: "would_mark_failed", + } + if attachment.Referenced { + issue.Kind = IssueReferencedMissing + issue.Action = "manual_repair_required" + } else if apply { + changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced( + ctx, attachment.ID, attachment.Status, cutoff, + "reconciler_missing_object", startedAt, + ) + if markError != nil { + return result, fmt.Errorf("mark missing attachment %q failed: %w", attachment.ID, markError) + } + if changed { + issue.Action = "marked_failed" + result.FailedAttachments++ + } else { + issue.Action = "skipped_concurrent_change" + } + } + result.Issues = append(result.Issues, issue) + + case attachment.Status == domain.AttachmentStatusPending: + issue := Issue{ + Kind: IssueStalePending, AttachmentID: attachment.ID, + ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced, + Action: "would_mark_failed", + } + if attachment.Referenced { + issue.Action = "manual_repair_required" + } else if apply { + changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced( + ctx, attachment.ID, attachment.Status, cutoff, + "reconciler_stale_pending", startedAt, + ) + if markError != nil { + return result, fmt.Errorf("mark stale attachment %q failed: %w", attachment.ID, markError) + } + if changed { + issue.Action = "marked_failed" + result.FailedAttachments++ + if objectExists { + if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil { + return result, fmt.Errorf("delete stale pending object %q: %w", attachment.ObjectKey, err) + } + result.DeletedObjects++ + } + } else { + issue.Action = "skipped_concurrent_change" + } + } + result.Issues = append(result.Issues, issue) + + case attachment.Status == domain.AttachmentStatusFailed && objectExists: + issue := Issue{ + Kind: IssueFailedObject, AttachmentID: attachment.ID, + ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced, + Action: "would_delete", + } + if attachment.Referenced { + issue.Action = "manual_repair_required" + } else if apply { + if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil { + return result, fmt.Errorf("delete failed attachment object %q: %w", attachment.ObjectKey, err) + } + issue.Action = "deleted" + result.DeletedObjects++ + } + result.Issues = append(result.Issues, issue) + } + } + + sort.Slice(result.Issues, func(i, j int) bool { + if result.Issues[i].ObjectKey == result.Issues[j].ObjectKey { + return result.Issues[i].Kind < result.Issues[j].Kind + } + return result.Issues[i].ObjectKey < result.Issues[j].ObjectKey + }) + result.Duration = r.now().UTC().Sub(startedAt) + result.DurationMillis = result.Duration.Milliseconds() + return result, nil +} diff --git a/internal/reconcile/reconciler_test.go b/internal/reconcile/reconciler_test.go new file mode 100644 index 0000000..8d0b84d --- /dev/null +++ b/internal/reconcile/reconciler_test.go @@ -0,0 +1,209 @@ +package reconcile + +import ( + "context" + "errors" + "testing" + "time" + + "mtusi2026lab4/internal/adapter/objectstore" + "mtusi2026lab4/internal/domain" +) + +type fakeMetadataStore struct { + records []AttachmentRecord + markError error + skipMark bool +} + +func (store *fakeMetadataStore) ListAttachmentInventory( + context.Context, +) ([]AttachmentRecord, error) { + return append([]AttachmentRecord(nil), store.records...), nil +} + +func (store *fakeMetadataStore) MarkAttachmentFailedIfUnreferenced( + _ context.Context, + attachmentID string, + expectedStatus domain.AttachmentStatus, + updatedBefore time.Time, + errorCode string, + updatedAt time.Time, +) (bool, error) { + if store.markError != nil { + return false, store.markError + } + if store.skipMark { + return false, nil + } + for index := range store.records { + record := &store.records[index] + if record.ID != attachmentID || record.Status != expectedStatus || + record.UpdatedAt.After(updatedBefore) || record.Referenced { + continue + } + record.Status = domain.AttachmentStatusFailed + record.UpdatedAt = updatedAt + _ = errorCode + return true, nil + } + return false, nil +} + +func TestReconcilerDoesNotDeleteAfterConcurrentMetadataChange(t *testing.T) { + old := time.Now().UTC().Add(-2 * time.Hour) + metadata := &fakeMetadataStore{ + records: []AttachmentRecord{{ + ID: "pending-race", ObjectKey: "attachments/pending-race", + Status: domain.AttachmentStatusPending, UpdatedAt: old, + }}, + skipMark: true, + } + objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{ + "attachments/pending-race": {Key: "attachments/pending-race", LastModified: old}, + }} + reconciler, err := New(metadata, objects) + if err != nil { + t.Fatal(err) + } + + result, err := reconciler.Run(context.Background(), time.Hour, true) + if err != nil { + t.Fatal(err) + } + if result.DeletedObjects != 0 || result.FailedAttachments != 0 { + t.Fatalf("concurrent metadata change must suppress deletion: %+v", result) + } + if _, exists := objects.objects["attachments/pending-race"]; !exists { + t.Fatal("reconciler deleted an object after losing the metadata race") + } + if len(result.Issues) != 1 || result.Issues[0].Action != "skipped_concurrent_change" { + t.Fatalf("unexpected race result: %+v", result.Issues) + } +} + +type fakeObjectInventory struct { + objects map[string]objectstore.StoredObject + deleteError error +} + +func (store *fakeObjectInventory) List( + context.Context, + string, +) ([]objectstore.StoredObject, error) { + objects := make([]objectstore.StoredObject, 0, len(store.objects)) + for _, object := range store.objects { + objects = append(objects, object) + } + return objects, nil +} + +func (store *fakeObjectInventory) Delete(_ context.Context, key string) error { + if store.deleteError != nil { + return store.deleteError + } + delete(store.objects, key) + return nil +} + +func TestReconcilerDryRunApplyAndRepeat(t *testing.T) { + now := time.Now().UTC() + old := now.Add(-2 * time.Hour) + fresh := now.Add(-5 * time.Minute) + metadata := &fakeMetadataStore{records: []AttachmentRecord{ + {ID: "ready-present", ObjectKey: "attachments/ready-present", Status: domain.AttachmentStatusReady, UpdatedAt: old}, + {ID: "ready-missing", ObjectKey: "attachments/ready-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old}, + {ID: "ready-used-missing", ObjectKey: "attachments/ready-used-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old, Referenced: true}, + {ID: "pending-old", ObjectKey: "attachments/pending-old", Status: domain.AttachmentStatusPending, UpdatedAt: old}, + {ID: "failed-old", ObjectKey: "attachments/failed-old", Status: domain.AttachmentStatusFailed, UpdatedAt: old}, + {ID: "pending-fresh", ObjectKey: "attachments/pending-fresh", Status: domain.AttachmentStatusPending, UpdatedAt: fresh}, + }} + objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{ + "attachments/ready-present": {Key: "attachments/ready-present", LastModified: old}, + "attachments/pending-old": {Key: "attachments/pending-old", LastModified: old}, + "attachments/failed-old": {Key: "attachments/failed-old", LastModified: old}, + "attachments/orphan-old": {Key: "attachments/orphan-old", LastModified: old}, + "attachments/orphan-fresh": {Key: "attachments/orphan-fresh", LastModified: fresh}, + }} + reconciler, err := New(metadata, objects) + if err != nil { + t.Fatal(err) + } + + dryRun, err := reconciler.Run(context.Background(), time.Hour, false) + if err != nil { + t.Fatal(err) + } + if !dryRun.DryRun || len(dryRun.Issues) != 5 || + dryRun.DeletedObjects != 0 || dryRun.FailedAttachments != 0 { + t.Fatalf("unexpected dry-run result: %+v", dryRun) + } + if len(objects.objects) != 5 { + t.Fatal("dry-run changed object storage") + } + for _, record := range metadata.records { + if record.ID == "ready-missing" && record.Status != domain.AttachmentStatusReady { + t.Fatal("dry-run changed metadata") + } + } + + applied, err := reconciler.Run(context.Background(), time.Hour, true) + if err != nil { + t.Fatal(err) + } + if applied.DeletedObjects != 3 || applied.FailedAttachments != 2 { + t.Fatalf("unexpected apply result: %+v", applied) + } + if _, exists := objects.objects["attachments/orphan-old"]; exists { + t.Fatal("old orphan was not deleted") + } + if _, exists := objects.objects["attachments/orphan-fresh"]; !exists { + t.Fatal("fresh orphan must be protected by grace period") + } + if _, exists := objects.objects["attachments/ready-present"]; !exists { + t.Fatal("healthy object was deleted") + } + for _, issue := range applied.Issues { + if issue.AttachmentID == "ready-used-missing" && + (!issue.Protected || issue.Action != "manual_repair_required") { + t.Fatalf("referenced attachment was not protected: %+v", issue) + } + } + + repeated, err := reconciler.Run(context.Background(), time.Hour, true) + if err != nil { + t.Fatal(err) + } + if repeated.DeletedObjects != 0 || repeated.FailedAttachments != 0 { + t.Fatalf("repeat must be mutation-idempotent: %+v", repeated) + } + if len(repeated.Issues) != 1 || !repeated.Issues[0].Protected { + t.Fatalf("repeat should retain only protected manual issue: %+v", repeated) + } +} + +func TestReconcilerReportsMutationFailures(t *testing.T) { + old := time.Now().UTC().Add(-2 * time.Hour) + metadata := &fakeMetadataStore{records: []AttachmentRecord{ + {ID: "missing", ObjectKey: "attachments/missing", Status: domain.AttachmentStatusReady, UpdatedAt: old}, + }} + metadata.markError = errors.New("injected database failure") + objects := &fakeObjectInventory{objects: make(map[string]objectstore.StoredObject)} + reconciler, err := New(metadata, objects) + if err != nil { + t.Fatal(err) + } + if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil { + t.Fatal("injected database mutation failure was hidden") + } + + metadata.records = nil + metadata.markError = nil + objects.objects["attachments/orphan"] = objectstore.StoredObject{ + Key: "attachments/orphan", LastModified: old, + } + objects.deleteError = errors.New("injected object-store failure") + if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil { + t.Fatal("injected object-store mutation failure was hidden") + } +} diff --git a/internal/transport/http/dto/attachment.go b/internal/transport/http/dto/attachment.go new file mode 100644 index 0000000..0f5b1c4 --- /dev/null +++ b/internal/transport/http/dto/attachment.go @@ -0,0 +1,59 @@ +package dto + +import ( + "time" + + "mtusi2026lab4/internal/domain" +) + +type AttachmentVariantResponse struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +type AttachmentResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status domain.AttachmentStatus `json:"status"` + Variants []AttachmentVariantResponse `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func AttachmentFromDomain(attachment domain.Attachment) AttachmentResponse { + variants := make([]AttachmentVariantResponse, 0, len(attachment.Variants)) + for _, variant := range attachment.Variants { + variants = append(variants, AttachmentVariantResponse{ + Name: variant.Name, + ContentType: variant.ContentType, + SizeBytes: variant.SizeBytes, + Width: variant.Width, + Height: variant.Height, + SHA256: variant.SHA256, + }) + } + return AttachmentResponse{ + ID: attachment.ID, + ChatID: attachment.ChatID, + OwnerID: attachment.OwnerID, + Filename: attachment.Filename, + ContentType: attachment.ContentType, + SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, + Status: attachment.Status, + Variants: variants, + ErrorCode: attachment.ErrorCode, + CreatedAt: attachment.CreatedAt, + UpdatedAt: attachment.UpdatedAt, + } +} diff --git a/internal/transport/http/dto/message.go b/internal/transport/http/dto/message.go index 051a6b6..d2c6031 100644 --- a/internal/transport/http/dto/message.go +++ b/internal/transport/http/dto/message.go @@ -7,15 +7,17 @@ import ( ) type CreateMessageRequest struct { - Text string `json:"text"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` } type MessageResponse struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt time.Time `json:"created_at"` } type MessagePageResponse struct { @@ -24,9 +26,14 @@ type MessagePageResponse struct { } func MessageFromDomain(message domain.Message) MessageResponse { + attachmentIDs := append([]string(nil), message.AttachmentIDs...) + if attachmentIDs == nil { + attachmentIDs = make([]string, 0) + } + return MessageResponse{ ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, - Text: message.Text, CreatedAt: message.CreatedAt, + Text: message.Text, AttachmentIDs: attachmentIDs, CreatedAt: message.CreatedAt, } } diff --git a/internal/transport/http/handlers/create_message.go b/internal/transport/http/handlers/create_message.go index 653f879..30581c7 100644 --- a/internal/transport/http/handlers/create_message.go +++ b/internal/transport/http/handlers/create_message.go @@ -44,6 +44,7 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { ActorID: currentUser.ID, ChatID: chatID, Text: request.Text, + AttachmentIDs: request.AttachmentIDs, Method: http.MethodPost, CanonicalPath: "/api/v1/chats/" + chatID + "/messages", IdempotencyKey: idempotencyKey, @@ -53,6 +54,11 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { "Idempotency-Key is already used for another request", requestID) return } + if errors.Is(err, application.ErrAttachmentNotReady) { + response.WriteError(w, http.StatusConflict, response.CodeConflict, + "attachment is not ready", requestID) + return + } if errors.Is(err, application.ErrInvalidInput) { response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest, strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": "), requestID) diff --git a/internal/transport/http/handlers/get_attachment.go b/internal/transport/http/handlers/get_attachment.go new file mode 100644 index 0000000..dccbc34 --- /dev/null +++ b/internal/transport/http/handlers/get_attachment.go @@ -0,0 +1,94 @@ +package handlers + +import ( + "errors" + "io" + "mime" + "net/http" + "strconv" + + "mtusi2026lab4/internal/application" + "mtusi2026lab4/internal/domain" + "mtusi2026lab4/internal/transport/http/dto" + "mtusi2026lab4/internal/transport/http/middleware" + "mtusi2026lab4/internal/transport/http/response" + + "github.com/google/uuid" +) + +func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.GetRequestID(r.Context()) + currentUser := middleware.GetCurrentUser(r.Context()) + attachmentID, ok := parseAttachmentID(w, r, requestID) + if !ok { + return + } + + attachment, err := h.service.GetAttachment(r.Context(), attachmentID, currentUser.ID) + if errors.Is(err, domain.ErrNotFound) { + response.WriteError(w, http.StatusNotFound, response.CodeNotFound, + "attachment not found", requestID) + return + } + if err != nil { + response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable, + "service is temporarily unavailable", requestID) + return + } + response.WriteJSON(w, http.StatusOK, dto.AttachmentFromDomain(attachment), requestID) +} + +func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.GetRequestID(r.Context()) + currentUser := middleware.GetCurrentUser(r.Context()) + attachmentID, ok := parseAttachmentID(w, r, requestID) + if !ok { + return + } + + download, err := h.service.DownloadAttachment( + r.Context(), attachmentID, currentUser.ID, r.URL.Query().Get("variant"), + ) + switch { + case errors.Is(err, domain.ErrNotFound): + response.WriteError(w, http.StatusNotFound, response.CodeNotFound, + "attachment or variant not found", requestID) + return + case errors.Is(err, application.ErrAttachmentNotReady): + response.WriteError(w, http.StatusConflict, response.CodeConflict, + "attachment is not ready", requestID) + return + case errors.Is(err, application.ErrStorageUnavailable): + response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable, + "object storage is temporarily unavailable", requestID) + return + case err != nil: + response.WriteError(w, http.StatusInternalServerError, response.CodeInternalError, + "unexpected internal error", requestID) + return + } + defer download.Body.Close() + + w.Header().Set("Content-Type", download.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(download.SizeBytes, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Disposition", mime.FormatMediaType( + "attachment", map[string]string{"filename": download.Filename}, + )) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, download.Body) +} + +func parseAttachmentID( + w http.ResponseWriter, + r *http.Request, + requestID string, +) (string, bool) { + parsedID, err := uuid.Parse(r.PathValue("attachment_id")) + if err != nil { + response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest, + "invalid attachment_id uuid", requestID) + return "", false + } + return parsedID.String(), true +} diff --git a/internal/transport/http/handlers/upload_attachment.go b/internal/transport/http/handlers/upload_attachment.go new file mode 100644 index 0000000..aa7fdf7 --- /dev/null +++ b/internal/transport/http/handlers/upload_attachment.go @@ -0,0 +1,97 @@ +package handlers + +import ( + "errors" + "mime" + "net/http" + "strings" + + "mtusi2026lab4/internal/application" + "mtusi2026lab4/internal/domain" + "mtusi2026lab4/internal/transport/http/dto" + "mtusi2026lab4/internal/transport/http/middleware" + "mtusi2026lab4/internal/transport/http/response" + + "github.com/google/uuid" +) + +func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.GetRequestID(r.Context()) + currentUser := middleware.GetCurrentUser(r.Context()) + + parsedChatID, err := uuid.Parse(r.PathValue("chat_id")) + if err != nil { + response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest, + "invalid chat_id uuid", requestID) + return + } + filename := r.URL.Query().Get("filename") + if filename == "" { + response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest, + "filename query parameter is required", requestID) + return + } + + contentType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || len(parameters) != 0 || + (contentType != "image/png" && contentType != "image/jpeg") { + response.WriteError(w, http.StatusUnsupportedMediaType, + response.CodeUnsupportedMediaType, + "Content-Type must be image/png or image/jpeg", requestID) + return + } + if r.ContentLength > domain.MaxAttachmentSizeBytes { + response.WriteError(w, http.StatusRequestEntityTooLarge, + response.CodePayloadTooLarge, + "attachment exceeds 10 MiB", requestID) + return + } + if r.Body == nil { + response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest, + "attachment body is required", requestID) + return + } + defer r.Body.Close() + + attachment, err := h.service.UploadAttachment( + r.Context(), + application.UploadAttachmentCommand{ + ActorID: currentUser.ID, + ChatID: parsedChatID.String(), + Filename: filename, + DeclaredContentType: contentType, + Body: r.Body, + }, + ) + switch { + case errors.Is(err, application.ErrPayloadTooLarge): + response.WriteError(w, http.StatusRequestEntityTooLarge, + response.CodePayloadTooLarge, + "attachment exceeds 10 MiB or 40 million pixels", requestID) + return + case errors.Is(err, application.ErrUnsupportedMediaType), errors.Is(err, application.ErrInvalidImage): + response.WriteError(w, http.StatusUnsupportedMediaType, + response.CodeUnsupportedMediaType, + "body is not a valid PNG/JPEG matching the declared Content-Type", requestID) + return + case errors.Is(err, application.ErrInvalidInput): + message := strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": ") + response.WriteError(w, http.StatusBadRequest, + response.CodeInvalidRequest, message, requestID) + return + case errors.Is(err, domain.ErrNotFound): + response.WriteError(w, http.StatusNotFound, + response.CodeNotFound, "chat not found", requestID) + return + case errors.Is(err, application.ErrStorageUnavailable): + response.WriteError(w, http.StatusServiceUnavailable, + response.CodeUnavailable, "object storage is temporarily unavailable", requestID) + return + case err != nil: + response.WriteError(w, http.StatusServiceUnavailable, + response.CodeUnavailable, "service is temporarily unavailable", requestID) + return + } + + response.WriteJSON(w, http.StatusCreated, dto.AttachmentFromDomain(attachment), requestID) +} diff --git a/migrations/000003_attachments.down.sql b/migrations/000003_attachments.down.sql new file mode 100644 index 0000000..4cc8393 --- /dev/null +++ b/migrations/000003_attachments.down.sql @@ -0,0 +1,36 @@ +BEGIN; + +-- Downgrading would destroy the only content of attachment-only messages. +-- Refuse instead of silently losing valid lab4 data. +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM messages + WHERE btrim(text) = '' + ) THEN + RAISE EXCEPTION + 'cannot downgrade while attachment-only messages exist'; + END IF; +END; +$$; + +DROP TRIGGER message_attachments_payload_check ON message_attachments; +DROP TRIGGER messages_payload_check ON messages; +DROP FUNCTION enforce_message_payload_from_link(); +DROP FUNCTION enforce_message_payload_from_message(); + +DROP TABLE message_attachments; +DROP TABLE attachments; + +ALTER TABLE messages + DROP CONSTRAINT messages_id_chat_unique, + DROP CONSTRAINT messages_text_length; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length + CHECK (char_length(text) BETWEEN 1 AND 4000), + ADD CONSTRAINT messages_text_not_blank + CHECK (btrim(text) <> ''); + +COMMIT; diff --git a/migrations/000003_attachments.up.sql b/migrations/000003_attachments.up.sql new file mode 100644 index 0000000..01d7d72 --- /dev/null +++ b/migrations/000003_attachments.up.sql @@ -0,0 +1,191 @@ +BEGIN; + +-- Text may be empty only when the same transaction links at least one +-- attachment. A deferred constraint trigger below enforces that invariant. +ALTER TABLE messages + DROP CONSTRAINT messages_text_length, + DROP CONSTRAINT messages_text_not_blank; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length + CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD CONSTRAINT messages_id_chat_unique + UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + object_key TEXT NOT NULL, + filename TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 BYTEA NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + status VARCHAR(16) NOT NULL, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT attachments_id_chat_unique + UNIQUE (id, chat_id), + + CONSTRAINT attachments_object_key_unique + UNIQUE (object_key), + + CONSTRAINT attachments_owner_membership_foreign_key + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) + ON DELETE RESTRICT, + + CONSTRAINT attachments_object_key_not_blank + CHECK (btrim(object_key) <> ''), + + CONSTRAINT attachments_filename_length + CHECK (char_length(filename) BETWEEN 1 AND 255), + + CONSTRAINT attachments_filename_safe + CHECK ( + btrim(filename) <> '' + AND filename NOT IN ('.', '..') + AND position('/' in filename) = 0 + AND position(chr(92) in filename) = 0 + AND filename !~ '[[:cntrl:]]' + ), + + CONSTRAINT attachments_content_type_supported + CHECK (content_type IN ('image/png', 'image/jpeg')), + + CONSTRAINT attachments_size_range + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT attachments_sha256_length + CHECK (octet_length(sha256) = 32), + + CONSTRAINT attachments_dimensions_positive + CHECK (width > 0 AND height > 0), + + CONSTRAINT attachments_pixel_limit + CHECK (width::BIGINT * height::BIGINT <= 40000000), + + CONSTRAINT attachments_status_supported + CHECK (status IN ('pending', 'ready', 'failed')), + + CONSTRAINT attachments_status_error_consistency + CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND btrim(error_code) <> '') + OR + (status IN ('pending', 'ready') AND error_code IS NULL) + ), + + CONSTRAINT attachments_updated_after_creation + CHECK (updated_at >= created_at) +); + +CREATE INDEX attachments_chat_created_at_id_idx + ON attachments (chat_id, created_at, id); + +CREATE INDEX attachments_status_updated_at_idx + ON attachments (status, updated_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL, + + CONSTRAINT message_attachments_primary_key + PRIMARY KEY (message_id, attachment_id), + + CONSTRAINT message_attachments_position_unique + UNIQUE (message_id, position), + + CONSTRAINT message_attachments_position_range + CHECK (position BETWEEN 0 AND 9), + + CONSTRAINT message_attachments_message_foreign_key + FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) + ON DELETE CASCADE, + + CONSTRAINT message_attachments_attachment_foreign_key + FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) + ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_id_idx + ON message_attachments (attachment_id, message_id); + +CREATE FUNCTION enforce_message_payload_from_message() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM messages AS message + WHERE message.id = NEW.id + AND btrim(message.text) = '' + AND NOT EXISTS ( + SELECT 1 + FROM message_attachments AS link + WHERE link.message_id = message.id + ) + ) THEN + RAISE EXCEPTION USING + ERRCODE = '23514', + CONSTRAINT = 'messages_content_required', + MESSAGE = 'message must contain non-blank text or an attachment'; + END IF; + RETURN NULL; +END; +$$; + +CREATE FUNCTION enforce_message_payload_from_link() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + affected_message_id UUID; +BEGIN + IF TG_OP = 'DELETE' THEN + affected_message_id := OLD.message_id; + ELSE + affected_message_id := NEW.message_id; + END IF; + + IF EXISTS ( + SELECT 1 + FROM messages AS message + WHERE message.id = affected_message_id + AND btrim(message.text) = '' + AND NOT EXISTS ( + SELECT 1 + FROM message_attachments AS link + WHERE link.message_id = message.id + ) + ) THEN + RAISE EXCEPTION USING + ERRCODE = '23514', + CONSTRAINT = 'messages_content_required', + MESSAGE = 'message must contain non-blank text or an attachment'; + END IF; + RETURN NULL; +END; +$$; + +CREATE CONSTRAINT TRIGGER messages_payload_check +AFTER INSERT OR UPDATE ON messages +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION enforce_message_payload_from_message(); + +CREATE CONSTRAINT TRIGGER message_attachments_payload_check +AFTER INSERT OR UPDATE OR DELETE ON message_attachments +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION enforce_message_payload_from_link(); + +COMMIT; diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..a0cfd3f --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu + +project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +secrets_dir="$project_root/secrets" + +command -v openssl >/dev/null 2>&1 || { + printf '%s\n' "openssl is required to generate local secrets" >&2 + exit 1 +} + +umask 077 +mkdir -p "$secrets_dir" +chmod 700 "$secrets_dir" + +write_if_missing() { + destination=$1 + value=$2 + if [ ! -e "$destination" ]; then + printf '%s' "$value" > "$destination" + fi + chmod 600 "$destination" +} + +random_hex() { + openssl rand -hex "$1" +} + +write_if_missing "$secrets_dir/postgres_password.txt" "$(random_hex 24)" +postgres_password=$(cat "$secrets_dir/postgres_password.txt") +write_if_missing "$secrets_dir/database_url.txt" \ + "postgres://messenger:${postgres_password}@postgres:5432/messenger?sslmode=disable" +write_if_missing "$secrets_dir/cursor_signing_key.txt" "$(random_hex 32)" +write_if_missing "$secrets_dir/minio_root_user.txt" "lab4-root" +write_if_missing "$secrets_dir/minio_root_password.txt" "$(random_hex 24)" +write_if_missing "$secrets_dir/s3_access_key_id.txt" "lab4-api" +write_if_missing "$secrets_dir/s3_secret_access_key.txt" "$(random_hex 24)" + +printf '%s\n' "Local secret files are present with mode 600; existing values were not overwritten." diff --git a/scripts/restart.sh b/scripts/restart.sh index fd04a05..539f12b 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -6,13 +6,20 @@ cd "$project_root" env_file=${ENV_FILE:-.env} compose() { - docker compose --env-file "$env_file" "$@" + if [ -f "$env_file" ]; then + docker compose --env-file "$env_file" "$@" + else + docker compose "$@" + fi } -# PostgreSQL хранит все обязательные данные и сессии в named volume. -# Redis содержит только восстанавливаемый кеш и намеренно не имеет volume. -compose restart postgres redis -compose up -d --wait postgres redis +# PostgreSQL хранит метаданные и сессии, MinIO — исходные байты вложений; +# оба используют named volumes. Redis содержит только восстанавливаемый кеш. +compose restart postgres redis minio +compose up -d --wait postgres redis minio + +# Повторный bootstrap приватного bucket/policy/user обязан быть идемпотентным. +compose run --rm minio-bootstrap >/dev/null # up восстанавливает желаемое число реплик, включая ранее вручную остановленную. compose up -d --no-deps api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 931cccc..5eb76ba 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -9,7 +9,15 @@ cd "$project_root" env_file=${ENV_FILE:-.env} matched_container= -for container_id in $(docker compose --env-file "$env_file" ps -a -q api); do +compose() { + if [ -f "$env_file" ]; then + docker compose --env-file "$env_file" "$@" + else + docker compose "$@" + fi +} + +for container_id in $(compose ps -a -q api); do instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then if [ -n "$matched_container" ]; then diff --git a/student_tests/application_helper_test.go b/student_tests/application_helper_test.go index 73abda2..4767c7b 100644 --- a/student_tests/application_helper_test.go +++ b/student_tests/application_helper_test.go @@ -8,7 +8,11 @@ import ( const testCursorSigningKey = "student-tests-cursor-signing-key-at-least-32-bytes" -func newTestService(repository application.Repository, sessionTTL time.Duration) *application.Service { +func newTestService( + repository application.Repository, + sessionTTL time.Duration, + options ...application.ServiceOption, +) *application.Service { if sessionTTL <= 0 { sessionTTL = time.Hour } @@ -16,7 +20,7 @@ func newTestService(repository application.Repository, sessionTTL time.Duration) if err != nil { panic(err) } - service, err := application.NewService(repository, codec, sessionTTL) + service, err := application.NewService(repository, codec, sessionTTL, options...) if err != nil { panic(err) } diff --git a/student_tests/attachment_upload_test.go b/student_tests/attachment_upload_test.go new file mode 100644 index 0000000..d961947 --- /dev/null +++ b/student_tests/attachment_upload_test.go @@ -0,0 +1,256 @@ +package student_tests + +import ( + "bytes" + "context" + "encoding/json" + "image" + "image/color" + "image/png" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + store "mtusi2026lab4/internal/adapter/memory" + "mtusi2026lab4/internal/application" + "mtusi2026lab4/internal/domain" + "mtusi2026lab4/internal/transport/http/dto" + "mtusi2026lab4/internal/transport/http/handlers" + "mtusi2026lab4/internal/transport/http/middleware" +) + +type httpFakeObjectStore struct { + mu sync.Mutex + objects map[string][]byte +} + +func (store *httpFakeObjectStore) Put(_ context.Context, object application.ObjectPut) error { + payload, err := io.ReadAll(object.Body) + if err != nil { + return err + } + store.mu.Lock() + store.objects[object.Key] = payload + store.mu.Unlock() + return nil +} + +func (store *httpFakeObjectStore) Get(_ context.Context, key string) (application.ObjectContent, error) { + store.mu.Lock() + defer store.mu.Unlock() + payload, exists := store.objects[key] + if !exists { + return application.ObjectContent{}, domain.ErrNotFound + } + copyOfPayload := append([]byte(nil), payload...) + return application.ObjectContent{ + Body: io.NopCloser(bytes.NewReader(copyOfPayload)), + SizeBytes: int64(len(copyOfPayload)), + }, nil +} + +func (store *httpFakeObjectStore) Delete(_ context.Context, key string) error { + store.mu.Lock() + delete(store.objects, key) + store.mu.Unlock() + return nil +} + +func setupAttachmentUploadServer( + t *testing.T, +) (*httptest.Server, *store.MemoryStore, *httpFakeObjectStore) { + t.Helper() + memoryStore := store.NewMemoryStore() + repository := store.NewMemoryRepository(memoryStore) + objectStore := &httpFakeObjectStore{objects: make(map[string][]byte)} + service := newTestService( + repository, + time.Hour, + application.WithObjectStore(objectStore), + ) + handler := handlers.NewHandler(service) + + mux := http.NewServeMux() + mux.HandleFunc( + "POST /api/v1/chats/{chat_id}/attachments", + requireLegacyAuth(memoryStore, handler.UploadAttachment), + ) + mux.HandleFunc( + "GET /api/v1/attachments/{attachment_id}", + requireLegacyAuth(memoryStore, handler.GetAttachment), + ) + mux.HandleFunc( + "GET /api/v1/attachments/{attachment_id}/content", + requireLegacyAuth(memoryStore, handler.DownloadAttachment), + ) + var root http.Handler = mux + root = middleware.Recovery(root) + root = middleware.RequestID(root) + return httptest.NewServer(root), memoryStore, objectStore +} + +func pngPayload(t *testing.T) []byte { + t.Helper() + var payload bytes.Buffer + img := image.NewRGBA(image.Rect(0, 0, 2, 2)) + img.Set(0, 0, color.RGBA{R: 255, A: 255}) + if err := png.Encode(&payload, img); err != nil { + t.Fatal(err) + } + return payload.Bytes() +} + +func uploadRequest( + t *testing.T, + baseURL string, + chatID string, + userID string, + filename string, + contentType string, + payload []byte, +) *http.Response { + t.Helper() + requestURL := baseURL + "/api/v1/chats/" + chatID + + "/attachments?filename=" + url.QueryEscape(filename) + request, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(payload)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", contentType) + request.Header.Set("X-User-Id", userID) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + return response +} + +func attachmentGetRequest(t *testing.T, requestURL string, userID string) *http.Response { + t.Helper() + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("X-User-Id", userID) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + return response +} + +func TestUploadAttachmentHTTPContract(t *testing.T) { + server, memoryStore, objectStore := setupAttachmentUploadServer(t) + defer server.Close() + user, err := memoryStore.CreateUser("attachment_user", "Attachment User") + if err != nil { + t.Fatal(err) + } + outsider, err := memoryStore.CreateUser("attachment_outsider", "Attachment Outsider") + if err != nil { + t.Fatal(err) + } + chat, err := memoryStore.CreateChat("Images", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + payload := pngPayload(t) + response := uploadRequest(t, server.URL, chat.ID, user.ID, "кот.png", "image/png", payload) + defer response.Body.Close() + if response.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(response.Body) + t.Fatalf("status = %d, want 201: %s", response.StatusCode, body) + } + var attachment dto.AttachmentResponse + if err = json.NewDecoder(response.Body).Decode(&attachment); err != nil { + t.Fatal(err) + } + if attachment.Status != domain.AttachmentStatusReady || attachment.Filename != "кот.png" { + t.Fatalf("unexpected response: %+v", attachment) + } + if attachment.Variants == nil || len(attachment.Variants) != 0 || attachment.ErrorCode != nil { + t.Fatalf("contract arrays/nulls differ: %+v", attachment) + } + objectStore.mu.Lock() + storedObjects := len(objectStore.objects) + objectStore.mu.Unlock() + if storedObjects != 1 { + t.Fatalf("stored objects = %d, want 1", storedObjects) + } + + metadataResponse := attachmentGetRequest(t, server.URL+"/api/v1/attachments/"+attachment.ID, user.ID) + defer metadataResponse.Body.Close() + if metadataResponse.StatusCode != http.StatusOK { + t.Fatalf("metadata status = %d, want 200", metadataResponse.StatusCode) + } + + contentResponse := attachmentGetRequest(t, + server.URL+"/api/v1/attachments/"+attachment.ID+"/content", user.ID) + defer contentResponse.Body.Close() + downloaded, err := io.ReadAll(contentResponse.Body) + if err != nil { + t.Fatal(err) + } + if contentResponse.StatusCode != http.StatusOK || !bytes.Equal(downloaded, payload) { + t.Fatalf("download status/content differ: status=%d", contentResponse.StatusCode) + } + if contentResponse.Header.Get("Content-Type") != "image/png" { + t.Fatalf("download Content-Type = %q", contentResponse.Header.Get("Content-Type")) + } + + for _, suffix := range []string{"", "/content"} { + outsiderResponse := attachmentGetRequest(t, + server.URL+"/api/v1/attachments/"+attachment.ID+suffix, outsider.ID) + _ = outsiderResponse.Body.Close() + if outsiderResponse.StatusCode != http.StatusNotFound { + t.Fatalf("outsider %q status = %d, want 404", suffix, outsiderResponse.StatusCode) + } + } + + missingVariant := attachmentGetRequest(t, + server.URL+"/api/v1/attachments/"+attachment.ID+"/content?variant=thumbnail", user.ID) + _ = missingVariant.Body.Close() + if missingVariant.StatusCode != http.StatusNotFound { + t.Fatalf("missing variant status = %d, want 404", missingVariant.StatusCode) + } +} + +func TestUploadAttachmentHTTPErrorMapping(t *testing.T) { + server, memoryStore, _ := setupAttachmentUploadServer(t) + defer server.Close() + user, _ := memoryStore.CreateUser("attachment_errors", "Attachment Errors") + chat, _ := memoryStore.CreateChat("Images", []string{user.ID}) + validPNG := pngPayload(t) + + testCases := []struct { + name string + chatID string + filename string + contentType string + payload []byte + wantStatus int + }{ + {name: "unsafe filename", chatID: chat.ID, filename: "../image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusBadRequest}, + {name: "unsupported content type", chatID: chat.ID, filename: "image.gif", contentType: "image/gif", payload: []byte("GIF89a"), wantStatus: http.StatusUnsupportedMediaType}, + {name: "MIME substitution", chatID: chat.ID, filename: "image.jpg", contentType: "image/jpeg", payload: validPNG, wantStatus: http.StatusUnsupportedMediaType}, + {name: "corrupt image", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: []byte("bad"), wantStatus: http.StatusUnsupportedMediaType}, + {name: "too large", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantStatus: http.StatusRequestEntityTooLarge}, + {name: "unknown chat", chatID: "11111111-1111-4111-8111-111111111111", filename: "image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusNotFound}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + response := uploadRequest(t, server.URL, testCase.chatID, user.ID, + testCase.filename, testCase.contentType, testCase.payload) + defer response.Body.Close() + if response.StatusCode != testCase.wantStatus { + body, _ := io.ReadAll(response.Body) + t.Fatalf("status = %d, want %d: %s", response.StatusCode, testCase.wantStatus, body) + } + }) + } +} diff --git a/student_tests/idempotency_test.go b/student_tests/idempotency_test.go index c026757..a8ffb28 100644 --- a/student_tests/idempotency_test.go +++ b/student_tests/idempotency_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "reflect" "sync" "testing" @@ -133,7 +134,7 @@ func TestMessageIdempotencyReplayAndConflict(t *testing.T) { first := decodeMessageResponse(t, firstPayload) replay := decodeMessageResponse(t, replayPayload) - if first != replay { + if !reflect.DeepEqual(first, replay) { t.Fatalf("replay changed message: first=%+v replay=%+v", first, replay) } @@ -256,7 +257,7 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) { for message := range results { if count == 0 { expected = message - } else if message != expected { + } else if !reflect.DeepEqual(message, expected) { t.Fatalf("concurrent replay returned different message: %+v != %+v", message, expected) } count++ @@ -273,3 +274,40 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) { t.Fatalf("expected exactly one stored message, got %d", len(messages)) } } + +func TestMessageIdempotencyTreatsMissingAndEmptyAttachmentsEqually(t *testing.T) { + _, serverURL, user, chatID := createMessageFixture(t) + + status, firstPayload := sendMessageRequest( + t, + serverURL, + chatID, + user.ID, + "default-attachments", + `{"text":"same semantic request"}`, + ) + if status != http.StatusCreated { + t.Fatalf("first request: expected 201, got %d: %s", status, firstPayload) + } + + status, replayPayload := sendMessageRequest( + t, + serverURL, + chatID, + user.ID, + "default-attachments", + `{"attachment_ids":[],"text":"same semantic request"}`, + ) + if status != http.StatusCreated { + t.Fatalf("replay: expected 201, got %d: %s", status, replayPayload) + } + + first := decodeMessageResponse(t, firstPayload) + replay := decodeMessageResponse(t, replayPayload) + if !reflect.DeepEqual(first, replay) { + t.Fatalf("missing attachment_ids and [] must replay identically") + } + if first.AttachmentIDs == nil || len(first.AttachmentIDs) != 0 { + t.Fatalf("response must contain attachment_ids: [], got %#v", first.AttachmentIDs) + } +} diff --git a/student_tests/message_attachments_test.go b/student_tests/message_attachments_test.go new file mode 100644 index 0000000..425cee0 --- /dev/null +++ b/student_tests/message_attachments_test.go @@ -0,0 +1,133 @@ +package student_tests + +import ( + "fmt" + "net/http" + "reflect" + "testing" + + "mtusi2026lab4/internal/domain" +) + +func TestMessageAttachmentContract(t *testing.T) { + memoryStore, serverURL, user, chatID := createMessageFixture(t) + + readyID := "11111111-1111-4111-8111-111111111111" + if err := memoryStore.PutAttachment(domain.Attachment{ + ID: readyID, + ChatID: chatID, + OwnerID: user.ID, + Status: domain.AttachmentStatusReady, + }); err != nil { + t.Fatalf("seed ready attachment: %v", err) + } + + status, payload := sendMessageRequest( + t, + serverURL, + chatID, + user.ID, + "attachment-only", + fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, readyID), + ) + if status != http.StatusCreated { + t.Fatalf("attachment-only message: expected 201, got %d: %s", status, payload) + } + + message := decodeMessageResponse(t, payload) + if message.Text != "" { + t.Fatalf("attachment-only message changed text: %q", message.Text) + } + if !reflect.DeepEqual(message.AttachmentIDs, []string{readyID}) { + t.Fatalf("attachment_ids changed: %#v", message.AttachmentIDs) + } + + items, _, err := memoryStore.GetMessages(chatID, user.ID, 10, nil) + if err != nil { + t.Fatalf("read stored messages: %v", err) + } + if len(items) != 1 || !reflect.DeepEqual(items[0].AttachmentIDs, []string{readyID}) { + t.Fatalf("stored attachment relation differs: %#v", items) + } +} + +func TestMessageAttachmentValidationAndAccess(t *testing.T) { + memoryStore, serverURL, user, chatID := createMessageFixture(t) + otherChat, err := memoryStore.CreateChat("Other chat", []string{user.ID}) + if err != nil { + t.Fatalf("create other chat: %v", err) + } + + pendingID := "22222222-2222-4222-8222-222222222222" + if err = memoryStore.PutAttachment(domain.Attachment{ + ID: pendingID, + ChatID: chatID, + OwnerID: user.ID, + Status: domain.AttachmentStatusPending, + }); err != nil { + t.Fatalf("seed pending attachment: %v", err) + } + + foreignID := "33333333-3333-4333-8333-333333333333" + if err = memoryStore.PutAttachment(domain.Attachment{ + ID: foreignID, + ChatID: otherChat.ID, + OwnerID: user.ID, + Status: domain.AttachmentStatusReady, + }); err != nil { + t.Fatalf("seed foreign attachment: %v", err) + } + + testCases := []struct { + name string + body string + wantStatus int + }{ + { + name: "empty without attachments", + body: `{"text":"","attachment_ids":[]}`, + wantStatus: http.StatusBadRequest, + }, + { + name: "invalid attachment uuid", + body: `{"text":"hello","attachment_ids":["not-a-uuid"]}`, + wantStatus: http.StatusBadRequest, + }, + { + name: "duplicate attachment", + body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q,%q]}`, pendingID, pendingID), + wantStatus: http.StatusBadRequest, + }, + { + name: "unknown attachment", + body: `{"text":"hello","attachment_ids":["44444444-4444-4444-8444-444444444444"]}`, + wantStatus: http.StatusNotFound, + }, + { + name: "attachment from another chat", + body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, foreignID), + wantStatus: http.StatusNotFound, + }, + { + name: "attachment not ready", + body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, pendingID), + wantStatus: http.StatusConflict, + }, + } + + for index, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + status, payload := sendMessageRequest( + t, + serverURL, + chatID, + user.ID, + fmt.Sprintf("attachment-validation-%d", index), + testCase.body, + ) + if status != testCase.wantStatus { + t.Fatalf("expected %d, got %d: %s", testCase.wantStatus, status, payload) + } + }) + } +} diff --git a/tests/attachment_boundaries.py b/tests/attachment_boundaries.py new file mode 100644 index 0000000..1cd272b --- /dev/null +++ b/tests/attachment_boundaries.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Black-box attachment validation and exact contract-boundary checks.""" + +import os +import struct +import zlib + +from attachment_memory import cleanup, noisy_png, png_chunk +from client import Client, check, run + + +MAX_BYTES = 10 * 1024 * 1024 + + +def exact_size_png(sample, target_size=MAX_BYTES): + check(sample.endswith(png_chunk(b"IEND", b"")), "sample PNG has no final IEND") + prefix, iend = sample[:-12], sample[-12:] + filler_size = target_size - len(prefix) - len(iend) - 12 + check(filler_size >= 0, "sample PNG is larger than boundary") + result = prefix + png_chunk(b"ruSt", bytes(filler_size)) + iend + check(len(result) == target_size, "exact-size PNG construction failed") + return result + + +def exact_pixel_png(width=10_000, height=4_000): + compressor = zlib.compressobj(level=9) + row = bytes(1 + width * 3) + parts = [] + for _ in range(height): + chunk = compressor.compress(row) + if chunk: + parts.append(chunk) + parts.append(compressor.flush()) + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + png_chunk(b"IDAT", b"".join(parts)) + + png_chunk(b"IEND", b"") + ) + + +def config_only_png(width, height): + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + ) + + +def upload(client, actor, chat, filename, payload, expected, schema=None): + return client.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename), + expected, + actor=actor, + raw=payload, + headers={"Content-Type": "image/png"}, + schema=schema, + ) + + +def main(): + client = Client(os.environ.get("BASE_URL"), None, timeout=30) + alice, bob, eve, chat = client.bootstrap() + users = [alice["id"], bob["id"], eve["id"]] + sample = open(os.path.join(os.path.dirname(__file__), "fixtures", "sample.png"), "rb").read() + + try: + boundary = exact_size_png(sample) + response = upload(client, alice, chat, "exact-10mib.png", boundary, 201, "Attachment") + attachment = response["json"] + check(attachment["size_bytes"] == MAX_BYTES, "10 MiB boundary size changed") + downloaded = client.call( + "GET", "/api/v1/attachments/%s/content" % attachment["id"], + 200, actor=alice, + )["bytes"] + check(downloaded == boundary, "10 MiB original changed after round trip") + + pixels = exact_pixel_png() + upload(client, alice, chat, "exact-40m-pixels.png", pixels, 201, "Attachment") + + upload(client, alice, chat, "over-pixels.png", config_only_png(20_000, 10_000), 413) + upload(client, alice, chat, "truncated.png", sample[:33], 415) + + corrupted = noisy_png() + check(len(corrupted) > 70_000, "corruption fixture is too small") + corrupted = bytearray(corrupted) + corrupted[70_000] ^= 0xff + upload(client, alice, chat, "corrupt-after-prefix.png", bytes(corrupted), 415) + + upload(client, alice, chat, "%20%20%20", sample, 400) + upload(client, alice, chat, "c1%C2%85.png", sample, 400) + upload(client, alice, chat, "invalid-%FF.png", sample, 400) + print("PASS: exact byte/pixel boundaries, complete decode, and filename validation are correct.") + finally: + cleanup(chat["id"], users) + + +if __name__ == "__main__": + run(main) diff --git a/tests/attachment_memory.py b/tests/attachment_memory.py new file mode 100644 index 0000000..fbe0597 --- /dev/null +++ b/tests/attachment_memory.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Concurrent real PNG uploads with Docker API-container memory sampling.""" + +import concurrent.futures +import json +import os +import struct +import subprocess +import threading +import time +import uuid +import zlib + +from client import Client, check, run + + +def png_chunk(kind, payload): + return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload)) + + +def noisy_png(width=2048, height=1024): + rows = bytearray() + for _ in range(height): + rows.append(0) + rows.extend(os.urandom(width * 3)) + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + png_chunk(b"IDAT", zlib.compress(bytes(rows), level=0)) + + png_chunk(b"IEND", b"") + ) + + +def bytes_from_docker(value): + number, unit = value.split()[0], value.split()[0][-3:] + if unit in {"KiB", "MiB", "GiB"}: + number = value.split()[0][:-3] + else: + unit = "B" + factor = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}[unit] + return int(float(number) * factor) + + +def api_container_ids(): + result = subprocess.run( + ["docker", "compose", "ps", "-q", "api"], + check=True, text=True, capture_output=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def sample_memory(container_ids, stop_event, peaks): + while not stop_event.is_set(): + result = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}", *container_ids], + check=False, text=True, capture_output=True, + ) + if result.returncode == 0: + for line in result.stdout.splitlines(): + name, usage = line.split("\t", 1) + current = bytes_from_docker(usage.split(" / ", 1)[0]) + peaks[name] = max(peaks.get(name, 0), current) + stop_event.wait(0.05) + + +def cleanup(chat_id, user_ids): + object_script = ( + 'root_user="$(cat /run/secrets/minio_root_user)"\n' + 'root_password="$(cat /run/secrets/minio_root_password)"\n' + 'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n' + 'mc rm --recursive --force "local/${S3_BUCKET}/attachments/' + chat_id + '/" >/dev/null 2>&1 || true\n' + ) + subprocess.run( + ["docker", "compose", "run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", object_script], + check=True, + ) + quoted_users = ",".join("'%s'" % value for value in user_ids) + sql = f""" +BEGIN; +DELETE FROM messages WHERE chat_id = '{chat_id}'; +DELETE FROM attachments WHERE chat_id = '{chat_id}'; +DELETE FROM sessions WHERE user_id IN ({quoted_users}); +DELETE FROM chat_members WHERE chat_id = '{chat_id}'; +DELETE FROM chats WHERE id = '{chat_id}'; +DELETE FROM users WHERE id IN ({quoted_users}); +COMMIT; +""" + subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger"], + input=sql, text=True, check=True, + ) + + +def main(): + concurrency = 12 + base_url = os.environ.get("BASE_URL", "http://127.0.0.1:8080") + client = Client(base_url, None, timeout=30) + alice, bob, eve, chat = client.bootstrap() + users = [alice["id"], bob["id"], eve["id"]] + payload = noisy_png() + stop_event = threading.Event() + peaks = {} + sampler = threading.Thread( + target=sample_memory, + args=(api_container_ids(), stop_event, peaks), + daemon=True, + ) + sampler.start() + started = time.monotonic() + + def upload(index): + response = client.call( + "POST", + "/api/v1/chats/%s/attachments?filename=memory-%02d.png" % (chat["id"], index), + 201, + actor=alice, + raw=payload, + headers={"Content-Type": "image/png", "X-Load-Id": uuid.uuid4().hex}, + schema="Attachment", + ) + return response["headers"]["X-Instance-Id"] + + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + instances = list(pool.map(upload, range(concurrency))) + elapsed = time.monotonic() - started + stop_event.set() + sampler.join(timeout=10) + check(len(set(instances)) >= 2, "uploads did not exercise both API replicas") + check(len(peaks) >= 2, "memory was not sampled for both API replicas") + summary = { + "concurrency": concurrency, + "payload_bytes": len(payload), + "total_uploaded_bytes": len(payload) * concurrency, + "elapsed_seconds": round(elapsed, 3), + "successful_uploads": len(instances), + "api_instances_observed": len(set(instances)), + "peak_memory_bytes_by_container": peaks, + } + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + print("PASS: concurrent attachment uploads remained within configured API memory limits.") + finally: + stop_event.set() + sampler.join(timeout=10) + cleanup(chat["id"], users) + + +if __name__ == "__main__": + run(main) diff --git a/tests/minio_policy.sh b/tests/minio_policy.sh new file mode 100755 index 0000000..e9a58d1 --- /dev/null +++ b/tests/minio_policy.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$project_root" + +api_containers=$(docker compose ps -q api) +if [ -z "$api_containers" ]; then + printf "%s\n" "no running API containers found" >&2 + exit 1 +fi +for container_id in $api_containers; do + docker exec "$container_id" sh -ec ' + test ! -e /run/secrets/minio_root_user + test ! -e /run/secrets/minio_root_password + ! env | grep -Eq "^MINIO_ROOT_(USER|PASSWORD)=" + ' +done + +docker compose run --rm --entrypoint /bin/sh minio-bootstrap -ec ' +root_user="$(cat /run/secrets/minio_root_user)" +root_password="$(cat /run/secrets/minio_root_password)" +api_user="$(cat /run/secrets/s3_access_key_id)" +api_password="$(cat /run/secrets/s3_secret_access_key)" +probe_key="policy-probe/object.txt" + +mc alias set admin http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null +mc alias set limited http://minio:9000 "$api_user" "$api_password" --api S3v4 >/dev/null + +cleanup() { + mc rm --force "admin/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1 || true + mc rb --force "admin/forbidden-policy-probe" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +printf private-probe | mc pipe "limited/${S3_BUCKET}/${probe_key}" >/dev/null +mc stat "limited/${S3_BUCKET}/${probe_key}" >/dev/null +mc cat "limited/${S3_BUCKET}/${probe_key}" >/dev/null + +if mc mb "limited/forbidden-policy-probe" >/dev/null 2>&1; then + printf "%s\n" "limited API credentials unexpectedly created another bucket" >&2 + exit 1 +fi +if wget -qO- "http://minio:9000/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1; then + printf "%s\n" "anonymous HTTP GET unexpectedly read a private object" >&2 + exit 1 +fi + +mc rm --force "limited/${S3_BUCKET}/${probe_key}" >/dev/null +trap - EXIT INT TERM +printf "%s\n" "PASS: API has no MinIO root credentials; limited policy permits object IO only in the private bucket; anonymous GET is denied." +' diff --git a/tests/reconcile_integration.py b/tests/reconcile_integration.py new file mode 100644 index 0000000..f94b1fb --- /dev/null +++ b/tests/reconcile_integration.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Inject isolated DB/S3 inconsistencies and prove reconciler dry-run/apply/repeat.""" + +import json +import os +import subprocess +import time + + +USER_ID = "90000000-0000-4000-8000-000000000001" +CHAT_ID = "90000000-0000-4000-8000-000000000002" +MISSING_ID = "90000000-0000-4000-8000-000000000011" +PROTECTED_ID = "90000000-0000-4000-8000-000000000012" +PENDING_ID = "90000000-0000-4000-8000-000000000013" +MESSAGE_ID = "90000000-0000-4000-8000-000000000021" +PREFIX = "attachments/reconcile-integration" +ORPHAN_KEY = PREFIX + "/orphan" +PENDING_KEY = PREFIX + "/pending" + + +def run(command, *, input_text=None, capture=False): + result = subprocess.run( + command, + input=input_text, + text=True, + check=False, + capture_output=capture, + env=os.environ.copy(), + ) + if result.returncode != 0: + details = result.stderr.strip() if capture else "see command output" + raise RuntimeError("command failed: %s: %s" % (command[0], details)) + return result.stdout if capture else "" + + +def compose(*arguments, capture=False, input_text=None): + return run( + ["docker", "compose", *arguments], + input_text=input_text, + capture=capture, + ) + + +def database(sql): + compose( + "exec", "-T", "postgres", + "psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger", + input_text=sql, + ) + + +def object_command(script): + compose( + "run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", + "-ec", script, + ) + + +def clean_objects(): + object_command( + 'root_user="$(cat /run/secrets/minio_root_user)"\n' + 'root_password="$(cat /run/secrets/minio_root_password)"\n' + 'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n' + 'mc rm --force "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null 2>&1 || true\n' + 'mc rm --force "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null 2>&1 || true\n' + ) + + +def clean_database(): + database(f""" +BEGIN; +DELETE FROM messages WHERE id = '{MESSAGE_ID}'; +DELETE FROM attachments WHERE id IN ('{MISSING_ID}', '{PROTECTED_ID}', '{PENDING_ID}'); +DELETE FROM chat_members WHERE chat_id = '{CHAT_ID}'; +DELETE FROM chats WHERE id = '{CHAT_ID}'; +DELETE FROM users WHERE id = '{USER_ID}'; +COMMIT; +""") + + +def inject_database(): + database(f""" +BEGIN; +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('{USER_ID}', 'reconcile_fixture', 'Reconcile Fixture', 'not-used'); +INSERT INTO chats (id, title) VALUES ('{CHAT_ID}', 'Reconcile Integration'); +INSERT INTO chat_members (chat_id, user_id, position) VALUES ('{CHAT_ID}', '{USER_ID}', 0); +INSERT INTO attachments ( + id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, width, height, status, created_at, updated_at +) VALUES + ('{MISSING_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/missing', 'missing.png', 'image/png', + 7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'), + ('{PROTECTED_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/protected-missing', 'protected.png', 'image/png', + 7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'), + ('{PENDING_ID}', '{CHAT_ID}', '{USER_ID}', '{PENDING_KEY}', 'pending.png', 'image/png', + 7, decode(repeat('00', 32), 'hex'), 1, 1, 'pending', now() - interval '2 hours', now() - interval '2 hours'); +INSERT INTO messages (id, chat_id, sender_id, text) +VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{USER_ID}', 'protected attachment reference'); +INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) +VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{PROTECTED_ID}', 0); +COMMIT; +""") + + +def inject_objects(): + object_command( + 'root_user="$(cat /run/secrets/minio_root_user)"\n' + 'root_password="$(cat /run/secrets/minio_root_password)"\n' + 'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n' + 'printf orphan | mc pipe "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null\n' + 'printf pending | mc pipe "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null\n' + ) + + +def reconcile(apply=False): + command = [ + "run", "--rm", "--entrypoint", "/app/reconciler", "api", + "--grace-period", "1s", + ] + if apply: + command.append("--apply") + output = compose(*command, capture=True) + return json.loads(output) + + +def kinds(result): + return {issue["kind"] for issue in result["issues"]} + + +def main(): + clean_objects() + clean_database() + try: + inject_database() + inject_objects() + time.sleep(2) + + dry_run = reconcile(False) + expected = { + "orphan_object", "missing_object", "referenced_missing_object", "stale_pending", + } + if not expected.issubset(kinds(dry_run)): + raise RuntimeError("dry-run did not detect all injected inconsistencies") + if dry_run["deleted_objects"] or dry_run["failed_attachments"]: + raise RuntimeError("dry-run performed mutations") + + applied = reconcile(True) + if applied["deleted_objects"] != 2 or applied["failed_attachments"] != 2: + raise RuntimeError("apply mutation counts differ: %s" % applied) + + repeated = reconcile(True) + if repeated["deleted_objects"] or repeated["failed_attachments"]: + raise RuntimeError("repeated apply was not mutation-idempotent") + protected = [issue for issue in repeated["issues"] if issue.get("protected")] + if len(protected) != 1 or protected[0]["action"] != "manual_repair_required": + raise RuntimeError("referenced missing attachment was not protected") + + print(json.dumps({ + "dry_run_issues": len(dry_run["issues"]), + "apply_deleted_objects": applied["deleted_objects"], + "apply_failed_attachments": applied["failed_attachments"], + "repeat_mutations": repeated["deleted_objects"] + repeated["failed_attachments"], + "protected_manual_issues": len(protected), + "scanned_attachments": applied["scanned_attachments"], + "scanned_objects": applied["scanned_objects"], + "duration_ms": applied["duration_ms"], + }, ensure_ascii=False)) + print("PASS: real PostgreSQL/MinIO reconciliation is dry-run safe, idempotent and reference-aware.") + finally: + clean_objects() + clean_database() + + +if __name__ == "__main__": + main() diff --git a/tests/reconcile_scale.py b/tests/reconcile_scale.py new file mode 100644 index 0000000..2860062 --- /dev/null +++ b/tests/reconcile_scale.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Measure real PostgreSQL/MinIO reconciler traversal at two inventory sizes.""" + +import json +import subprocess +import time + + +PREFIX = "attachments/reconcile-scale" + + +def compose(*arguments, capture=False): + result = subprocess.run( + ["docker", "compose", *arguments], + check=False, + text=True, + capture_output=capture, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() if capture else "docker compose command failed") + return result.stdout if capture else "" + + +def object_command(script): + compose("run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", script) + + +def clean(): + object_command( + 'root_user="$(cat /run/secrets/minio_root_user)"\n' + 'root_password="$(cat /run/secrets/minio_root_password)"\n' + 'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n' + 'mc rm --recursive --force "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null 2>&1 || true\n' + ) + + +def create(count): + object_command( + 'root_user="$(cat /run/secrets/minio_root_user)"\n' + 'root_password="$(cat /run/secrets/minio_root_password)"\n' + 'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n' + 'mkdir -p /tmp/reconcile-scale\n' + 'index=0\n' + 'while [ "$index" -lt ' + str(count) + ' ]; do\n' + ' printf x > "/tmp/reconcile-scale/object-${index}"\n' + ' index=$((index + 1))\n' + 'done\n' + 'mc cp --recursive /tmp/reconcile-scale/ "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null\n' + ) + + +def dry_run(): + started = time.monotonic() + output = compose( + "run", "--rm", "--entrypoint", "/app/reconciler", "api", + "--grace-period", "1s", + capture=True, + ) + elapsed = time.monotonic() - started + result = json.loads(output) + return { + "scanned_attachments": result["scanned_attachments"], + "scanned_objects": result["scanned_objects"], + "issues": len(result["issues"]), + "reconciler_duration_ms": result["duration_ms"], + "container_start_and_run_ms": round(elapsed * 1000), + } + + +def main(): + measurements = [] + clean() + try: + for count in (100, 1000): + clean() + create(count) + time.sleep(2) + measurement = dry_run() + measurement["injected_orphan_objects"] = count + if measurement["issues"] < count: + raise RuntimeError("reconciler did not observe all scale fixtures") + measurements.append(measurement) + print(json.dumps({"measurements": measurements}, ensure_ascii=False)) + print("PASS: reconciler traversal cost measured on real PostgreSQL/MinIO inventories.") + finally: + clean() + + +if __name__ == "__main__": + main() diff --git a/tests/storage_outage.py b/tests/storage_outage.py new file mode 100644 index 0000000..85f7445 --- /dev/null +++ b/tests/storage_outage.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Verify contract responses and original durability during a real MinIO outage.""" + +import json +import os +import subprocess +import time + +from attachment_memory import cleanup +from client import Client, Failure, check, run + + +def compose(*arguments, **kwargs): + return subprocess.run(["docker", "compose", *arguments], check=True, **kwargs) + + +def wait_for_status(client, path, expected, timeout=20): + deadline = time.monotonic() + timeout + last_error = None + while time.monotonic() < deadline: + try: + return client.call("GET", path, expected=expected) + except Failure as error: + last_error = error + time.sleep(0.25) + raise Failure("%s did not reach %d: %s" % (path, expected, last_error)) + + +def restore_storage(client): + compose("up", "-d", "--wait", "minio") + compose("run", "--rm", "minio-bootstrap") + wait_for_status(client, "/health/ready", 200, timeout=30) + + +def main(): + client = Client(os.environ.get("BASE_URL"), None, timeout=15) + alice, bob, eve, chat = client.bootstrap() + users = [alice["id"], bob["id"], eve["id"]] + attachment, original = client.attachment(alice, chat) + storage_stopped = False + + try: + compose("stop", "minio") + storage_stopped = True + readiness = wait_for_status(client, "/health/ready", 503) + check(readiness["elapsed"] < 5, "readiness exceeded its bounded timeout") + + started = time.monotonic() + failure = client.call( + "POST", + "/api/v1/chats/%s/attachments?filename=outage.png" % chat["id"], + 503, + actor=alice, + raw=original, + headers={"Content-Type": "image/png"}, + ) + elapsed = time.monotonic() - started + check(failure["json"]["error"]["code"] == "unavailable", "wrong S3 outage error code") + check(elapsed < 15, "upload outage response exceeded API/client timeout") + + restore_storage(client) + storage_stopped = False + downloaded = client.call( + "GET", "/api/v1/attachments/%s/content" % attachment["id"], + 200, actor=alice, + )["bytes"] + check(downloaded == original, "existing original changed across MinIO outage") + print(json.dumps({ + "readiness_503_seconds": round(readiness["elapsed"], 3), + "upload_503_seconds": round(elapsed, 3), + "original_bytes_verified": len(original), + }, sort_keys=True)) + print("PASS: MinIO outage returns bounded JSON 503 and preserves existing originals.") + finally: + if storage_stopped: + restore_storage(client) + cleanup(chat["id"], users) + + +if __name__ == "__main__": + run(main)