forked from mtusi2026/mtusi2026lab4
лаба 4 на оценку 5
This commit is contained in:
+6
-2
@@ -1,7 +1,8 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_INTERNAL_PORT=8080
|
||||
PUBLIC_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
@@ -11,10 +12,13 @@ SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_OPERATION_TIMEOUT=5s
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Keep the Go soft heap limit below the Compose hard limit (256 MiB by default).
|
||||
API_GO_MEMORY_LIMIT=220MiB
|
||||
|
||||
+3
-1
@@ -10,6 +10,7 @@ COPY internal/ ./internal/
|
||||
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/server ./cmd/server/main.go
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/reconciler ./cmd/reconciler/main.go
|
||||
|
||||
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
|
||||
|
||||
@@ -19,6 +20,7 @@ RUN addgroup -g 1000 appgroup && \
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /build/server /app/server
|
||||
COPY --from=builder /build/reconciler /app/reconciler
|
||||
|
||||
RUN chown -R appuser:appgroup /app
|
||||
|
||||
@@ -30,4 +32,4 @@ HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=10 \
|
||||
CMD wget -qO- http://127.0.0.1:8080/health/ready || exit 1
|
||||
|
||||
|
||||
ENTRYPOINT ["/app/server"]
|
||||
ENTRYPOINT ["/app/server"]
|
||||
|
||||
@@ -2,13 +2,17 @@ PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
RECONCILE_GRACE ?= 15m
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help init-secrets check validate test load test-attachment-boundaries test-attachment-memory test-storage-outage test-minio-policy test-persistence test-failover test-reconcile test-reconcile-scale reconcile-dry-run reconcile-apply
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
init-secrets:
|
||||
./scripts/init-secrets.sh
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
@@ -21,6 +25,18 @@ test:
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachment-memory:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_memory.py
|
||||
|
||||
test-attachment-boundaries:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_boundaries.py
|
||||
|
||||
test-storage-outage:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) tests/storage_outage.py
|
||||
|
||||
test-minio-policy:
|
||||
./tests/minio_policy.sh
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
@@ -28,3 +44,15 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile:
|
||||
$(PYTHON) tests/reconcile_integration.py
|
||||
|
||||
test-reconcile-scale:
|
||||
$(PYTHON) tests/reconcile_scale.py
|
||||
|
||||
reconcile-dry-run:
|
||||
docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)"
|
||||
|
||||
reconcile-apply:
|
||||
docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)" --apply
|
||||
|
||||
@@ -1,37 +1,123 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: TODO — заполнить перед сдачей.
|
||||
- Целевая оценка: **5**, выбран **трек 2 — согласованность PostgreSQL и S3**.
|
||||
- Commit/PR: TODO после публикации.
|
||||
- Использованные источники и библиотеки: Go standard library, `pgx/v5`, `minio-go/v7`, официальный исходный код MinIO и `mc`, Redis Go client. OpenAI Codex использовался для анализа контракта, проектирования, реализации и тестов; все утверждения ниже проверены перечисленными воспроизводимыми командами.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker с Compose v2, GNU Make, Python 3.9+, OpenSSL. Локально проверено в WSL2. MinIO и `mc` собираются из официальных Go-модулей в [infrastructure/minio/Dockerfile](infrastructure/minio/Dockerfile): сервер `RELEASE.2025-10-15T17-29-55Z`, клиент `RELEASE.2025-08-13T08-35-41Z`. Базовые образы закреплены digest.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
make init-secrets
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
docker compose --env-file .env ps
|
||||
BASE_URL=http://127.0.0.1:8080 make test
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
`make init-secrets` создаёт недостающие файлы с mode `600` и никогда не перезаписывает существующие значения. Секреты находятся в игнорируемом каталоге `secrets/` и не передаются через переменные окружения контейнера. Если `8080` занят, задаётся `PUBLIC_PORT`, например `PUBLIC_PORT=18080`; внутренний `APP_INTERNAL_PORT=8080` от него не зависит.
|
||||
|
||||
Безопасная остановка сохраняет named volumes:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Намеренное удаление всех учебных данных выполняется только явно:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Путь запроса: клиент → HAProxy → две API-реплики → PostgreSQL (источник истины метаданных/сессий) + приватный MinIO bucket (байты); Redis остаётся восстанавливаемым кешем чатов. У каждой API-реплики нет постоянного каталога оригиналов. Во время синхронного upload тело ограниченно копируется в файл OS temp, файл имеет случайное имя, удаляется в `defer` и нужен только для двукратной проверки/потокового S3 PUT.
|
||||
|
||||
Из lab3 сохранены два API-инстанса, HAProxy, PostgreSQL, Redis fallback, bearer-сессии, idempotency сообщений, курсоры, graceful shutdown и persistence/failover hooks. В lab4 добавлены:
|
||||
|
||||
- `Attachment` со статусами `pending/ready/failed`, SHA-256, истинным MIME, размером и размерами изображения;
|
||||
- таблицы `attachments` и `message_attachments`, same-chat foreign keys и deferred constraint «текст или вложение»;
|
||||
- `attachment_ids` во всех слоях Message без нарушения старого JSON (`[]` по умолчанию);
|
||||
- binary `POST`, metadata `GET` и авторизованный streaming `GET /content` без redirect;
|
||||
- общий MinIO volume, idempotent bootstrap приватного bucket и отдельный API user с минимальной policy;
|
||||
- readiness, зависящий от PostgreSQL и MinIO;
|
||||
- one-shot reconciler с dry-run по умолчанию, `--apply`, grace period и условными DB updates.
|
||||
|
||||
Upload возвращает `201 ready` только после последовательности: полное ограниченное чтение → проверка MIME/PNG/JPEG/пикселей и SHA-256 → insert `pending` → успешный S3 PUT → update `ready`. Ошибка PUT удаляет возможный объект и фиксирует `failed`. Ошибка ответа на финальный DB update считается неоднозначной: update мог уже закоммититься, поэтому объект не удаляется. API делает best-effort readback и либо восстанавливает успешный `201`, либо возвращает `503`, сохранив metadata+object для reconciler. Это исключает состояние `ready` без оригинала при потере ответа PostgreSQL.
|
||||
|
||||
Лимиты применяются до неограниченной работы: body читается через `LimitReader(10 MiB + 1)`, `DecodeConfig` проверяет максимум 40 млн пикселей до полного `Decode`, а полный `Decode` проверяет поток/CRC/IDAT/SOS до S3 PUT. Filename не участвует в object key и проверяется на valid UTF-8/blank/path/C0+C1 control characters. Object key формируется сервером как `attachments/{chat_id}/{attachment_id}/original`. На реплику допускаются не более четырёх upload pipeline и одного полного decode одновременно; `GOMEMLIMIT=220MiB` ниже container limit 256 MiB. S3 operation timeout — 5 s, HTTP read/write timeout — 15 s, HAProxy server/client timeout — 30 s, DB query timeout задаётся конфигурацией. Компенсация сначала получает отдельный 1 s budget для DB transition и лишь затем отдельный 1 s budget для best-effort S3 delete.
|
||||
|
||||
Публичный `/health/ready` проверяет PostgreSQL и MinIO и возвращает контрактный JSON `503`, если зависимость недоступна. HAProxy использует отдельный внутренний `/health/lb`, отражающий жизнеспособность/drain процесса, поэтому при общем отказе MinIO запросы не заменяются generic HTML-ошибкой балансировщика, а доходят до API и получают согласованный `503 unavailable`.
|
||||
|
||||
Соответствие ключевых переменных: `PUBLIC_PORT` — host port HAProxy; `APP_INTERNAL_PORT` — порт API в сети Compose; `DATABASE_URL_FILE`, `CURSOR_SIGNING_KEY_FILE`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` — Docker secrets; `S3_ENDPOINT=http://minio:9000`; `S3_BUCKET=messenger-attachments`; `S3_OPERATION_TIMEOUT=5s`.
|
||||
|
||||
## Механизм согласованности и трек 5
|
||||
|
||||
Гипотеза: периодический inventory PostgreSQL и MinIO с консервативным grace period способен повторяемо исправлять безопасные orphan/missing состояния, не удаляя свежие объекты и вложения, уже связанные с сообщениями.
|
||||
|
||||
Reconciler сравнивает все `attachments/` objects с метаданными:
|
||||
|
||||
- старый object без metadata удаляется только в apply;
|
||||
- старый `pending` очищает object и условно становится `failed`;
|
||||
- старый `ready` без object условно становится `failed`, только если не используется сообщением;
|
||||
- object у старого `failed` удаляется, только если запись не используется;
|
||||
- свежие записи/objects игнорируются, а referenced missing помечается `manual_repair_required` без мутации;
|
||||
- SQL update берёт row lock, затем отдельным statement на свежем READ COMMITTED snapshot повторно проверяет status, cutoff и отсутствие ссылки; concurrent message/reconciler race либо создаёт ссылку, либо переводит attachment в failed, но не оба состояния одновременно;
|
||||
- object stale-pending удаляется только после успешного условного DB transition, поэтому проигранная гонка приводит к `skipped_concurrent_change`, а не к потере готового оригинала.
|
||||
|
||||
Команды оператора:
|
||||
|
||||
```sh
|
||||
make reconcile-dry-run RECONCILE_GRACE=15m
|
||||
# Просмотреть JSON; затем при согласии:
|
||||
make reconcile-apply RECONCILE_GRACE=15m
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload/message/download/checksum/outsider 404 | [evidence/lab4-results.md](evidence/lab4-results.md) |
|
||||
| Границы/полная декодировка | `make test-attachment-boundaries` | Ровно 10 MiB и 40 млн пикселей приняты; truncated/corrupt-after-64-KiB отклонены 415; whitespace/C1 filename — 400 | [tests/attachment_boundaries.py](tests/attachment_boundaries.py) |
|
||||
| Failover API | `make test-failover` с указанными в evidence env | 142 успешных записи, 0 ошибок, recovery 2.771 s; файл читается другой репликой | [evidence/lab4-results.md](evidence/lab4-results.md) |
|
||||
| Restart и volumes | `make test-persistence` | Сессия, метаданные и bytes пережили restart PostgreSQL/Redis/MinIO/API/frontend/HAProxy | [scripts/restart.sh](scripts/restart.sh) |
|
||||
| Файловые негативные тесты | `go test ./...` | oversized bytes/pixels, invalid IDAT, truncated PNG/JPEG, corruption after 64 KiB, MIME substitution, unsafe filename, interrupted body, foreign attachment/chat | [internal/application/attachments_test.go](internal/application/attachments_test.go) |
|
||||
| Неоднозначный DB COMMIT | `go test ./internal/application -count=1` | Потеря ответа после commit восстанавливается readback; при одновременном read outage ready metadata сохраняет оригинал | [internal/application/attachments_test.go](internal/application/attachments_test.go) |
|
||||
| Реальный отказ MinIO | `make test-storage-outage` | readiness 503 за 2.004 s, upload 503 за 6.016 s; после recovery прежний original неизменен | [tests/storage_outage.py](tests/storage_outage.py) |
|
||||
| RAM при concurrency ≥10 | `make test-attachment-memory` | 12 × 6,293,028 B, 12/12 success; пики 31,216,107 и 26,214,400 B | [tests/attachment_memory.py](tests/attachment_memory.py) |
|
||||
| Private bucket / API policy | `make test-minio-policy` | Root secrets/env отсутствуют в API; limited user не создаёт bucket; anonymous GET запрещён | [tests/minio_policy.sh](tests/minio_policy.sh) |
|
||||
| Reconciler dry-run/apply/repeat | `make test-reconcile` | 4 issues; 2 objects deleted; 2 metadata failed; repeat 0 mutations; referenced защищён | [tests/reconcile_integration.py](tests/reconcile_integration.py) |
|
||||
| Стоимость обхода | `make test-reconcile-scale` | 100 objects: 9 ms; 1,000 objects: 38 ms core duration | [tests/reconcile_scale.py](tests/reconcile_scale.py) |
|
||||
| Static/race gates | `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS | [evidence/lab4-results.md](evidence/lab4-results.md) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Memory experiment: один прогон, 12 concurrent uploads, каждый PNG 2048×1024 и 6,293,028 bytes; суммарно 75,516,336 bytes. HAProxy распределил запросы по двум репликам. Docker stats опрашивался во время прогона. 12/12 завершились за 0.599 s, наблюдаемые пики — 29.8 и 25.0 MiB, существенно ниже container limit 256 MiB. После измерения отдельный chat prefix и синтетические DB rows удалены.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Reconciler integration: в отдельном namespace созданы old orphan object, old pending metadata+object, unreferenced ready metadata без object и referenced ready metadata без object. Dry-run нашёл четыре состояния и не мутировал данные. Apply удалил два objects и перевёл две записи в `failed`. Повторный apply сделал ноль мутаций; referenced missing остался ручной задачей. Синтетические данные удалены в `finally`.
|
||||
|
||||
Traversal experiment использовал один и тот же стенд после прогрева, один прогон на размер, отдельный prefix и однобайтовые objects. 100 injected orphans: 109 objects total, 9 ms core и 1,978 ms с запуском one-shot container. 1,000: 1,009 total, 38 ms core и 2,019 ms end-to-end. Это подтверждает повторяемость методики и ожидаемую близкую к линейной стоимость лишь на малом локальном масштабе.
|
||||
|
||||
| Воздействие | Ожидание | Наблюдение | Потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Stop одной API-реплики | Вторая продолжает запись/чтение общего файла | recovery 2.771 s, 142 writes, 0 errors | Нет |
|
||||
| Restart общего стенда | Named volumes сохраняют DB и S3 | persistence PASS | Нет |
|
||||
| S3 PUT error | 503, metadata failed, object отсутствует | Unit injection PASS | Нет ready-данных |
|
||||
| MinIO недоступен | readiness/upload дают JSON 503 до API/LB timeout | 2.004 s / 6.016 s, реальный stop/recovery PASS | Старый original неизменен |
|
||||
| DB finalize response потерян после COMMIT | Readback возвращает ready; при DB outage object не удаляется | Два unit fault-теста PASS | Ready не остаётся без original |
|
||||
| Orphan/missing/stale состояния | Dry-run, безопасный apply, repeat | Integration PASS | Referenced missing не изменён |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Inventory materializes metadata и object list в RAM; для большого bucket нужны pagination/checkpoints и пакетная выдача issues.
|
||||
- Один reconciler предполагается запускать оператором; перед scheduler нужны PostgreSQL advisory lock/lease и метрики длительности/ошибок.
|
||||
- `referenced_missing_object` намеренно не чинится автоматически: требуется восстановление bytes из backup или операторское решение.
|
||||
- Один-node MinIO и локальные измерения не доказывают HA самого object storage; это учебный стенд.
|
||||
- Experiment содержит по одному измерению на размер без статистических повторов; для production нужны warm-up, серии, p50/p95/p99, CPU/IO и существенно больший inventory.
|
||||
- Оригинал проверяется SHA-256 при upload и клиентом в smoke; каждый download не пересчитывает hash на лету.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/adapter/objectstore"
|
||||
"mtusi2026lab4/internal/adapter/postgres"
|
||||
"mtusi2026lab4/internal/config"
|
||||
"mtusi2026lab4/internal/database"
|
||||
"mtusi2026lab4/internal/reconcile"
|
||||
)
|
||||
|
||||
func main() {
|
||||
apply := flag.Bool("apply", false, "apply safe repairs; default is dry-run")
|
||||
gracePeriod := flag.Duration("grace-period", 15*time.Minute,
|
||||
"ignore metadata and objects newer than this duration")
|
||||
timeout := flag.Duration("timeout", 2*time.Minute, "maximum reconciliation runtime")
|
||||
flag.Parse()
|
||||
|
||||
if *gracePeriod <= 0 || *timeout <= 0 {
|
||||
log.Fatal("[FATAL] grace-period and timeout must be positive")
|
||||
}
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Invalid configuration: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||
defer cancel()
|
||||
databasePool, err := database.Open(ctx, cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] PostgreSQL initialization failed: %v", err)
|
||||
}
|
||||
defer databasePool.Close()
|
||||
metadataStore := postgres.NewPostgreSQLStore(databasePool, cfg.DatabaseQueryTimeout)
|
||||
|
||||
objectStore, err := objectstore.NewMinIOStore(
|
||||
cfg.S3Endpoint,
|
||||
cfg.S3Region,
|
||||
cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID,
|
||||
cfg.S3SecretAccessKey,
|
||||
cfg.S3OperationTimeout,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Invalid object storage configuration: %v", err)
|
||||
}
|
||||
if err = objectStore.Ping(ctx); err != nil {
|
||||
log.Fatalf("[FATAL] Object storage unavailable: %v", err)
|
||||
}
|
||||
|
||||
reconciler, err := reconcile.New(metadataStore, objectStore)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Reconciler initialization failed: %v", err)
|
||||
}
|
||||
result, err := reconciler.Run(ctx, *gracePeriod, *apply)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Reconciliation failed: %v", err)
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err = encoder.Encode(result); err != nil {
|
||||
log.Fatalf("[FATAL] Encode reconciliation result: %v", err)
|
||||
}
|
||||
if !*apply && len(result.Issues) > 0 {
|
||||
fmt.Fprintln(os.Stderr, "dry-run found reconciliation issues; rerun with --apply after review")
|
||||
}
|
||||
}
|
||||
+36
-1
@@ -11,6 +11,8 @@ import (
|
||||
"time"
|
||||
|
||||
cacheadapter "mtusi2026lab4/internal/adapter/cache"
|
||||
"mtusi2026lab4/internal/adapter/health"
|
||||
"mtusi2026lab4/internal/adapter/objectstore"
|
||||
"mtusi2026lab4/internal/adapter/postgres"
|
||||
redisadapter "mtusi2026lab4/internal/adapter/redis"
|
||||
"mtusi2026lab4/internal/application"
|
||||
@@ -43,6 +45,25 @@ func main() {
|
||||
databasePool,
|
||||
cfg.DatabaseQueryTimeout,
|
||||
)
|
||||
minioStore, err := objectstore.NewMinIOStore(
|
||||
cfg.S3Endpoint,
|
||||
cfg.S3Region,
|
||||
cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID,
|
||||
cfg.S3SecretAccessKey,
|
||||
cfg.S3OperationTimeout,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Invalid object storage configuration: %v", err)
|
||||
}
|
||||
storageContext, storageCancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
cfg.S3OperationTimeout,
|
||||
)
|
||||
if err = minioStore.Ping(storageContext); err != nil {
|
||||
log.Printf("[WARN] Object storage unavailable at startup: %v", err)
|
||||
}
|
||||
storageCancel()
|
||||
|
||||
var applicationStore application.Repository = postgresStore
|
||||
|
||||
@@ -90,6 +111,7 @@ func main() {
|
||||
cursorCodec,
|
||||
cfg.SessionTTL,
|
||||
application.WithIdempotencyTTL(cfg.IdempotencyTTL),
|
||||
application.WithObjectStore(minioStore),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Application initialization failed: %v", err)
|
||||
@@ -98,7 +120,7 @@ func main() {
|
||||
handler := handlers.NewHandler(
|
||||
service,
|
||||
handlers.WithReadiness(
|
||||
databasePool,
|
||||
health.NewMultiChecker(databasePool, minioStore),
|
||||
cfg.ReadinessTimeout,
|
||||
),
|
||||
handlers.WithSessionCookieSecure(cfg.SessionCookieSecure),
|
||||
@@ -107,6 +129,7 @@ func main() {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("GET /health/live", handler.HealthLive)
|
||||
mux.HandleFunc("GET /health/lb", handler.HealthLive)
|
||||
mux.HandleFunc("GET /health/ready", handler.HealthReady)
|
||||
|
||||
mux.HandleFunc(
|
||||
@@ -150,6 +173,18 @@ func main() {
|
||||
"GET /api/v1/chats/{chat_id}/messages",
|
||||
requireAuth(handler.GetMessages),
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
requireAuth(handler.UploadAttachment),
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"GET /api/v1/attachments/{attachment_id}",
|
||||
requireAuth(handler.GetAttachment),
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"GET /api/v1/attachments/{attachment_id}/content",
|
||||
requireAuth(handler.DownloadAttachment),
|
||||
)
|
||||
|
||||
var rootHandler http.Handler = mux
|
||||
rootHandler = middleware.Recovery(rootHandler)
|
||||
|
||||
+94
-2
@@ -5,7 +5,7 @@ services:
|
||||
dockerfile: infrastructure/haproxy/Dockerfile
|
||||
image: mtusi2026lab4-load-balancer
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${PUBLIC_PORT:-8080}:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
@@ -128,13 +128,84 @@ services:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
|
||||
minio:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infrastructure/minio/Dockerfile
|
||||
target: server
|
||||
args:
|
||||
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
|
||||
image: mtusi2026lab4-minio:RELEASE.2025-10-15T17-29-55Z
|
||||
command: ["server", "/data", "--address", ":9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
expose:
|
||||
- "9000"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.75"
|
||||
memory: 512M
|
||||
|
||||
minio-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infrastructure/minio/Dockerfile
|
||||
target: client
|
||||
args:
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
image: mtusi2026lab4-mc:RELEASE.2025-08-13T08-35-41Z
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
root_user="$$(cat /run/secrets/minio_root_user)"
|
||||
root_password="$$(cat /run/secrets/minio_root_password)"
|
||||
api_user="$$(cat /run/secrets/s3_access_key_id)"
|
||||
api_password="$$(cat /run/secrets/s3_secret_access_key)"
|
||||
mc alias set local http://minio:9000 "$$root_user" "$$root_password" --api S3v4
|
||||
mc mb --ignore-existing "local/$${S3_BUCKET}"
|
||||
mc anonymous set none "local/$${S3_BUCKET}"
|
||||
sed "s/__BUCKET__/$${S3_BUCKET}/g" /config/api-policy.template.json > /tmp/api-policy.json
|
||||
mc admin policy create local messenger-api /tmp/api-policy.json
|
||||
if ! mc admin user info local "$$api_user" >/dev/null 2>&1; then
|
||||
mc admin user add local "$$api_user" "$$api_password"
|
||||
fi
|
||||
mc admin policy attach local messenger-api --user "$$api_user"
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infrastructure/minio/api-policy.template.json:/config/api-policy.template.json:ro
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
APP_PORT: ${APP_INTERNAL_PORT:-8080}
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
@@ -148,6 +219,13 @@ services:
|
||||
CACHE_OPERATION_TIMEOUT: ${CACHE_OPERATION_TIMEOUT:-250ms}
|
||||
CACHE_POOL_SIZE: ${CACHE_POOL_SIZE:-10}
|
||||
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
|
||||
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
|
||||
S3_OPERATION_TIMEOUT: ${S3_OPERATION_TIMEOUT:-5s}
|
||||
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
@@ -161,8 +239,11 @@ services:
|
||||
READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s}
|
||||
SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s}
|
||||
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s}
|
||||
GOMEMLIMIT: ${API_GO_MEMORY_LIMIT:-220MiB}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- cursor_signing_key
|
||||
expose:
|
||||
- "8080"
|
||||
@@ -173,6 +254,8 @@ services:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
@@ -240,6 +323,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_insight_data:
|
||||
minio_data:
|
||||
postgres_restore_data:
|
||||
|
||||
secrets:
|
||||
@@ -249,3 +333,11 @@ secrets:
|
||||
file: ./secrets/database_url.txt
|
||||
cursor_signing_key:
|
||||
file: ./secrets/cursor_signing_key.txt
|
||||
minio_root_user:
|
||||
file: ./secrets/minio_root_user.txt
|
||||
minio_root_password:
|
||||
file: ./secrets/minio_root_password.txt
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key.txt
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Lab 4 verification summary
|
||||
|
||||
Date: 2026-10-29. Environment: local Docker Compose on WSL2; two API replicas; PostgreSQL 16.15; Redis 8.10.2; MinIO release `RELEASE.2025-10-15T17-29-55Z` built from the official source module.
|
||||
|
||||
| Check | Command | Observed result |
|
||||
| --- | --- | --- |
|
||||
| Public contract | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload, attachment-only message, authorized byte download, checksum/content type and outsider 404 |
|
||||
| Exact boundaries / decode | `BASE_URL=http://127.0.0.1:18080 make test-attachment-boundaries` | Exact 10 MiB and 40,000,000 pixels accepted; truncated/corrupted-after-64-KiB PNG rejected 415; blank/C1 filename rejected 400 |
|
||||
| One-replica failover | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/stop-one.sh make test-failover` | 2 instances before failure; 142 successful writes; 0 failed attempts; first recovery 2.771 s; attachment unchanged |
|
||||
| Full persistence restart | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/restart.sh make test-persistence` | PASS after restarting PostgreSQL, Redis, MinIO, both API replicas, frontend and HAProxy; existing session, metadata and bytes survived |
|
||||
| Real MinIO outage | `BASE_URL=http://127.0.0.1:18080 make test-storage-outage` | Public readiness returned JSON 503 in 2.004 s, upload returned JSON 503 in 6.016 s; after recovery the 111,817-byte previous original was byte-identical |
|
||||
| Private bucket / least privilege | `make test-minio-policy` | API containers had no MinIO root env/secrets; limited credentials could put/read/delete only in configured bucket; creating another bucket and anonymous GET failed |
|
||||
| Concurrent upload memory | `BASE_URL=http://127.0.0.1:18080 make test-attachment-memory` | 12/12 successful 6,293,028-byte PNG uploads (75,516,336 bytes total) in 0.599 s; peak API memory 31,216,107 and 26,214,400 bytes |
|
||||
| Reconciler behavior | `make test-reconcile` | Dry-run found 4 injected issues; apply deleted 2 objects and marked 2 attachments failed; repeat made 0 mutations; 1 referenced missing attachment remained protected/manual |
|
||||
| Reconciler traversal | `make test-reconcile-scale` | 100 orphan objects: 109 objects scanned, 9 ms core / 1,978 ms including container startup. 1,000 orphans: 1,009 scanned, 38 ms core / 2,019 ms including startup |
|
||||
| Ambiguous DB COMMIT fault | `go test ./internal/application -count=1` | Successful finalize with lost response is recovered by readback; if DB remains unavailable, ready metadata keeps its object and returns an honest error |
|
||||
| Go quality gates | `go test ./...`; `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS |
|
||||
|
||||
The measurements are single local runs, not production capacity claims. The reconciler currently materializes both inventories in memory, so its measured near-linear small-scale cost must be re-evaluated before using it with a large bucket.
|
||||
@@ -143,7 +143,12 @@ export const api = {
|
||||
);
|
||||
},
|
||||
|
||||
sendMessage(chatID: string, text: string, idempotencyKey: string): Promise<Message> {
|
||||
sendMessage(
|
||||
chatID: string,
|
||||
text: string,
|
||||
idempotencyKey: string,
|
||||
attachmentIDs: string[] = [],
|
||||
): Promise<Message> {
|
||||
return request<Message>(
|
||||
`${browserApi}/chats/${encodeURIComponent(chatID)}/messages`,
|
||||
{
|
||||
@@ -151,7 +156,7 @@ export const api = {
|
||||
headers: {
|
||||
"Idempotency-Key": idempotencyKey,
|
||||
},
|
||||
body: { text },
|
||||
body: { text, attachment_ids: attachmentIDs },
|
||||
},
|
||||
);
|
||||
},
|
||||
|
||||
@@ -24,6 +24,7 @@ export type Message = {
|
||||
chat_id: string;
|
||||
sender_id: string;
|
||||
text: string;
|
||||
attachment_ids: string[];
|
||||
created_at: string;
|
||||
};
|
||||
|
||||
|
||||
@@ -5,17 +5,31 @@ go 1.27.1
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.uber.org/atomic v1.12.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
|
||||
@@ -5,6 +5,9 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
@@ -15,16 +18,42 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
|
||||
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
|
||||
@@ -33,6 +62,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
@@ -40,4 +71,7 @@ golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -43,7 +43,7 @@ frontend public
|
||||
backend api_backend
|
||||
balance leastconn
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 8 api:8080
|
||||
@@ -55,7 +55,7 @@ backend message_backend
|
||||
option http-buffer-request
|
||||
retries 3
|
||||
retry-on conn-failure empty-response response-timeout 502 503 504
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 8 api:8080
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS minio-builder
|
||||
|
||||
ARG MINIO_VERSION=RELEASE.2025-10-15T17-29-55Z
|
||||
RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/minio@${MINIO_VERSION}
|
||||
|
||||
FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS mc-builder
|
||||
|
||||
ARG MC_VERSION=RELEASE.2025-08-13T08-35-41Z
|
||||
RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/mc@${MC_VERSION}
|
||||
|
||||
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS server
|
||||
|
||||
RUN apk add --no-cache ca-certificates wget && \
|
||||
addgroup -g 1000 minio && \
|
||||
adduser -u 1000 -G minio -s /sbin/nologin -D minio && \
|
||||
mkdir -p /data && \
|
||||
chown minio:minio /data
|
||||
|
||||
COPY --from=minio-builder /out/minio /usr/local/bin/minio
|
||||
|
||||
USER minio:minio
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 9000
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=20 \
|
||||
CMD wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/minio"]
|
||||
CMD ["server", "/data", "--address", ":9000"]
|
||||
|
||||
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS client
|
||||
|
||||
RUN apk add --no-cache ca-certificates && \
|
||||
addgroup -g 1000 minio && \
|
||||
adduser -u 1000 -G minio -s /sbin/nologin -D minio
|
||||
|
||||
COPY --from=mc-builder /out/mc /usr/local/bin/mc
|
||||
|
||||
USER minio:minio
|
||||
ENTRYPOINT ["/usr/local/bin/mc"]
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__BUCKET__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__BUCKET__/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Checker interface {
|
||||
Ping(ctx context.Context) error
|
||||
}
|
||||
|
||||
type MultiChecker struct {
|
||||
checkers []Checker
|
||||
}
|
||||
|
||||
func NewMultiChecker(checkers ...Checker) *MultiChecker {
|
||||
filtered := make([]Checker, 0, len(checkers))
|
||||
for _, checker := range checkers {
|
||||
if checker != nil {
|
||||
filtered = append(filtered, checker)
|
||||
}
|
||||
}
|
||||
return &MultiChecker{checkers: filtered}
|
||||
}
|
||||
|
||||
func (checker *MultiChecker) Ping(ctx context.Context) error {
|
||||
for index, dependency := range checker.checkers {
|
||||
if err := dependency.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("dependency %d: %w", index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -166,6 +166,15 @@ func (r *MemoryRepository) CreateMessageIdempotently(
|
||||
if _, err := r.legacy.GetChatByID(write.ChatID, write.ActorID); err != nil {
|
||||
return domain.Message{}, err
|
||||
}
|
||||
for _, attachmentID := range write.AttachmentIDs {
|
||||
attachment, err := r.legacy.GetAttachmentByID(attachmentID)
|
||||
if err != nil || attachment.ChatID != write.ChatID {
|
||||
return domain.Message{}, ErrNotFound
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusReady {
|
||||
return domain.Message{}, application.ErrAttachmentNotReady
|
||||
}
|
||||
}
|
||||
|
||||
if stored, exists := r.idempotencyByScope[scope]; exists {
|
||||
if stored.expiresAt.After(now) {
|
||||
@@ -177,7 +186,12 @@ func (r *MemoryRepository) CreateMessageIdempotently(
|
||||
delete(r.idempotencyByScope, scope)
|
||||
}
|
||||
|
||||
message, err := r.legacy.CreateMessage(write.ChatID, write.ActorID, write.Text)
|
||||
message, err := r.legacy.CreateMessage(
|
||||
write.ChatID,
|
||||
write.ActorID,
|
||||
write.Text,
|
||||
write.AttachmentIDs,
|
||||
)
|
||||
if err != nil {
|
||||
return domain.Message{}, err
|
||||
}
|
||||
@@ -277,6 +291,74 @@ func (r *MemoryRepository) DeleteSession(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MemoryRepository) CreateAttachment(
|
||||
ctx context.Context,
|
||||
attachment domain.Attachment,
|
||||
) (domain.Attachment, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
if err := r.legacy.PutAttachment(attachment); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (r *MemoryRepository) MarkAttachmentReady(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
updatedAt time.Time,
|
||||
) (domain.Attachment, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
return r.legacy.UpdateAttachmentStatus(
|
||||
attachmentID,
|
||||
domain.AttachmentStatusReady,
|
||||
nil,
|
||||
updatedAt,
|
||||
)
|
||||
}
|
||||
|
||||
func (r *MemoryRepository) MarkAttachmentFailed(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
errorCode string,
|
||||
updatedAt time.Time,
|
||||
) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := r.legacy.UpdateAttachmentStatus(
|
||||
attachmentID,
|
||||
domain.AttachmentStatusFailed,
|
||||
&errorCode,
|
||||
updatedAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *MemoryRepository) GetAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
callerID string,
|
||||
) (domain.Attachment, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
attachment, err := r.legacy.GetAttachmentByID(attachmentID)
|
||||
if err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
if _, err = r.legacy.GetChatByID(attachment.ChatID, callerID); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
if attachment.Variants == nil {
|
||||
attachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func memoryTokenHashKey(tokenHash []byte) ([32]byte, error) {
|
||||
var key [32]byte
|
||||
|
||||
|
||||
@@ -20,6 +20,8 @@ type MemoryStore struct {
|
||||
chatsByID map[string]domain.Chat //key = chat UUID
|
||||
|
||||
messagesByChatID map[string][]domain.Message // key = chat UUID, value = слайс сообщений
|
||||
|
||||
attachmentsByID map[string]domain.Attachment
|
||||
}
|
||||
|
||||
func NewMemoryStore() *MemoryStore {
|
||||
@@ -28,6 +30,7 @@ func NewMemoryStore() *MemoryStore {
|
||||
usersByUsername: make(map[string]string),
|
||||
chatsByID: make(map[string]domain.Chat),
|
||||
messagesByChatID: make(map[string][]domain.Message),
|
||||
attachmentsByID: make(map[string]domain.Attachment),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,12 +180,86 @@ func (s *MemoryStore) GetChatsForUser(
|
||||
return items, end < len(chats), nil
|
||||
}
|
||||
|
||||
// PutAttachment stores metadata for adapter and application tests. Binary
|
||||
// content intentionally does not belong to this in-memory metadata store.
|
||||
func (s *MemoryStore) PutAttachment(attachment domain.Attachment) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if _, exists := s.attachmentsByID[attachment.ID]; exists {
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
chat, exists := s.chatsByID[attachment.ChatID]
|
||||
if !exists {
|
||||
return ErrNotFound
|
||||
}
|
||||
|
||||
ownerIsMember := false
|
||||
for _, memberID := range chat.MemberIDs {
|
||||
if memberID == attachment.OwnerID {
|
||||
ownerIsMember = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ownerIsMember {
|
||||
return ErrNotFound
|
||||
}
|
||||
|
||||
s.attachmentsByID[attachment.ID] = attachment
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) GetAttachmentByID(
|
||||
attachmentID string,
|
||||
) (domain.Attachment, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
attachment, exists := s.attachmentsByID[attachmentID]
|
||||
if !exists {
|
||||
return domain.Attachment{}, ErrNotFound
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) UpdateAttachmentStatus(
|
||||
attachmentID string,
|
||||
status domain.AttachmentStatus,
|
||||
errorCode *string,
|
||||
updatedAt time.Time,
|
||||
) (domain.Attachment, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
attachment, exists := s.attachmentsByID[attachmentID]
|
||||
if !exists {
|
||||
return domain.Attachment{}, ErrNotFound
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusPending {
|
||||
return domain.Attachment{}, ErrConflict
|
||||
}
|
||||
attachment.Status = status
|
||||
attachment.ErrorCode = errorCode
|
||||
attachment.UpdatedAt = updatedAt
|
||||
if attachment.Variants == nil {
|
||||
attachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
}
|
||||
s.attachmentsByID[attachmentID] = attachment
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
// ---Сообщения---
|
||||
|
||||
func (s *MemoryStore) CreateMessage(
|
||||
chatID, senderID, text string,
|
||||
attachmentLists ...[]string,
|
||||
) (domain.Message, error) {
|
||||
messageID := uuid.NewString()
|
||||
attachmentIDs := make([]string, 0)
|
||||
if len(attachmentLists) > 0 {
|
||||
attachmentIDs = append(attachmentIDs, attachmentLists[0]...)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -216,11 +293,12 @@ func (s *MemoryStore) CreateMessage(
|
||||
}
|
||||
|
||||
message := domain.Message{
|
||||
ID: messageID,
|
||||
ChatID: chatID,
|
||||
SenderID: senderID,
|
||||
Text: text,
|
||||
CreatedAt: createdAt,
|
||||
ID: messageID,
|
||||
ChatID: chatID,
|
||||
SenderID: senderID,
|
||||
Text: text,
|
||||
AttachmentIDs: attachmentIDs,
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
|
||||
s.messagesByChatID[chatID] = append(messages, message)
|
||||
@@ -282,6 +360,12 @@ func (s *MemoryStore) GetMessages(
|
||||
|
||||
// копируем только возвращаемую страницу, а не всю историю, изменяя внутренний срез
|
||||
items := make([]domain.Message, end-start)
|
||||
copy(items, messages[start:end])
|
||||
for index, message := range messages[start:end] {
|
||||
message.AttachmentIDs = append(
|
||||
make([]string, 0, len(message.AttachmentIDs)),
|
||||
message.AttachmentIDs...,
|
||||
)
|
||||
items[index] = message
|
||||
}
|
||||
return items, end < len(messages), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/application"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type MinIOStore struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
operationTimeout time.Duration
|
||||
}
|
||||
|
||||
type StoredObject struct {
|
||||
Key string
|
||||
SizeBytes int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func NewMinIOStore(
|
||||
endpoint string,
|
||||
region string,
|
||||
bucket string,
|
||||
accessKeyID string,
|
||||
secretAccessKey string,
|
||||
operationTimeout time.Duration,
|
||||
) (*MinIOStore, error) {
|
||||
parsedEndpoint, err := url.Parse(endpoint)
|
||||
if err != nil || parsedEndpoint.Host == "" {
|
||||
return nil, fmt.Errorf("S3 endpoint must be an absolute HTTP(S) URL")
|
||||
}
|
||||
if parsedEndpoint.Scheme != "http" && parsedEndpoint.Scheme != "https" {
|
||||
return nil, fmt.Errorf("S3 endpoint scheme must be http or https")
|
||||
}
|
||||
if parsedEndpoint.Path != "" && parsedEndpoint.Path != "/" {
|
||||
return nil, fmt.Errorf("S3 endpoint must not contain a path")
|
||||
}
|
||||
if parsedEndpoint.RawQuery != "" || parsedEndpoint.Fragment != "" || parsedEndpoint.User != nil {
|
||||
return nil, fmt.Errorf("S3 endpoint must not contain credentials, query or fragment")
|
||||
}
|
||||
if strings.TrimSpace(bucket) == "" || strings.TrimSpace(accessKeyID) == "" ||
|
||||
strings.TrimSpace(secretAccessKey) == "" {
|
||||
return nil, fmt.Errorf("S3 bucket and credentials are required")
|
||||
}
|
||||
if operationTimeout <= 0 {
|
||||
return nil, fmt.Errorf("S3 operation timeout must be positive")
|
||||
}
|
||||
|
||||
client, err := minio.New(parsedEndpoint.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: parsedEndpoint.Scheme == "https",
|
||||
Region: region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create MinIO client: %w", err)
|
||||
}
|
||||
return &MinIOStore{client: client, bucket: bucket, operationTimeout: operationTimeout}, nil
|
||||
}
|
||||
|
||||
func (store *MinIOStore) Put(ctx context.Context, object application.ObjectPut) error {
|
||||
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
|
||||
defer cancel()
|
||||
upload, err := store.client.PutObject(
|
||||
operationContext,
|
||||
store.bucket,
|
||||
object.Key,
|
||||
object.Body,
|
||||
object.SizeBytes,
|
||||
minio.PutObjectOptions{ContentType: object.ContentType},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("put MinIO object: %w", err)
|
||||
}
|
||||
if upload.Size != object.SizeBytes {
|
||||
return fmt.Errorf("put MinIO object: stored %d bytes, want %d", upload.Size, object.SizeBytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MinIOStore) Get(
|
||||
ctx context.Context,
|
||||
key string,
|
||||
) (application.ObjectContent, error) {
|
||||
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
|
||||
stat, err := store.client.StatObject(
|
||||
operationContext,
|
||||
store.bucket,
|
||||
key,
|
||||
minio.StatObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return application.ObjectContent{}, fmt.Errorf("stat MinIO object: %w", err)
|
||||
}
|
||||
object, err := store.client.GetObject(
|
||||
operationContext,
|
||||
store.bucket,
|
||||
key,
|
||||
minio.GetObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return application.ObjectContent{}, fmt.Errorf("get MinIO object: %w", err)
|
||||
}
|
||||
firstByte := make([]byte, 1)
|
||||
readCount, readError := object.Read(firstByte)
|
||||
if readError != nil && readCount == 0 {
|
||||
_ = object.Close()
|
||||
cancel()
|
||||
return application.ObjectContent{}, fmt.Errorf("read MinIO object: %w", readError)
|
||||
}
|
||||
if readCount != 1 {
|
||||
_ = object.Close()
|
||||
cancel()
|
||||
return application.ObjectContent{}, fmt.Errorf("read MinIO object: expected non-empty object")
|
||||
}
|
||||
return application.ObjectContent{
|
||||
Body: &cancelingReadCloser{
|
||||
Reader: io.MultiReader(bytes.NewReader(firstByte), object),
|
||||
Closer: object,
|
||||
cancel: cancel,
|
||||
},
|
||||
SizeBytes: stat.Size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store *MinIOStore) List(
|
||||
ctx context.Context,
|
||||
prefix string,
|
||||
) ([]StoredObject, error) {
|
||||
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
|
||||
defer cancel()
|
||||
|
||||
objects := make([]StoredObject, 0)
|
||||
for object := range store.client.ListObjects(
|
||||
operationContext,
|
||||
store.bucket,
|
||||
minio.ListObjectsOptions{Prefix: prefix, Recursive: true},
|
||||
) {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("list MinIO objects: %w", object.Err)
|
||||
}
|
||||
objects = append(objects, StoredObject{
|
||||
Key: object.Key,
|
||||
SizeBytes: object.Size,
|
||||
LastModified: object.LastModified.UTC(),
|
||||
})
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (store *MinIOStore) Delete(ctx context.Context, key string) error {
|
||||
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
|
||||
defer cancel()
|
||||
if err := store.client.RemoveObject(
|
||||
operationContext,
|
||||
store.bucket,
|
||||
key,
|
||||
minio.RemoveObjectOptions{},
|
||||
); err != nil {
|
||||
return fmt.Errorf("delete MinIO object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MinIOStore) Ping(ctx context.Context) error {
|
||||
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
|
||||
defer cancel()
|
||||
exists, err := store.client.BucketExists(operationContext, store.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check MinIO bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("MinIO bucket %q does not exist", store.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type cancelingReadCloser struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (reader *cancelingReadCloser) Close() error {
|
||||
err := reader.Closer.Close()
|
||||
reader.cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
var _ application.ObjectStore = (*MinIOStore)(nil)
|
||||
@@ -0,0 +1,37 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewMinIOStoreValidatesConfiguration(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
bucket string
|
||||
accessKey string
|
||||
secretKey string
|
||||
timeout time.Duration
|
||||
wantError bool
|
||||
}{
|
||||
{name: "valid HTTP", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second},
|
||||
{name: "valid HTTPS", endpoint: "https://s3.example.test", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second},
|
||||
{name: "relative endpoint", endpoint: "minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
|
||||
{name: "endpoint path", endpoint: "http://minio:9000/path", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
|
||||
{name: "embedded credentials", endpoint: "http://user:pass@minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
|
||||
{name: "empty bucket", endpoint: "http://minio:9000", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
|
||||
{name: "empty credentials", endpoint: "http://minio:9000", bucket: "attachments", timeout: time.Second, wantError: true},
|
||||
{name: "invalid timeout", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", wantError: true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, err := NewMinIOStore(testCase.endpoint, "us-east-1", testCase.bucket,
|
||||
testCase.accessKey, testCase.secretKey, testCase.timeout)
|
||||
if (err != nil) != testCase.wantError {
|
||||
t.Fatalf("NewMinIOStore() error = %v, wantError %v", err, testCase.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
"mtusi2026lab4/internal/reconcile"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (s *PostgreSQLStore) CreateAttachment(
|
||||
ctx context.Context,
|
||||
attachment domain.Attachment,
|
||||
) (domain.Attachment, error) {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
err := s.pool.QueryRow(
|
||||
queryContext,
|
||||
`
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, object_key, filename, content_type,
|
||||
size_bytes, sha256, width, height, status, error_code,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, decode($8, 'hex'), $9, $10, $11, $12,
|
||||
$13, $14
|
||||
)
|
||||
RETURNING created_at, updated_at
|
||||
`,
|
||||
attachment.ID,
|
||||
attachment.ChatID,
|
||||
attachment.OwnerID,
|
||||
attachment.ObjectKey,
|
||||
attachment.Filename,
|
||||
attachment.ContentType,
|
||||
attachment.SizeBytes,
|
||||
attachment.SHA256,
|
||||
attachment.Width,
|
||||
attachment.Height,
|
||||
attachment.Status,
|
||||
attachment.ErrorCode,
|
||||
attachment.CreatedAt,
|
||||
attachment.UpdatedAt,
|
||||
).Scan(&attachment.CreatedAt, &attachment.UpdatedAt)
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("insert attachment metadata: %w", err)
|
||||
}
|
||||
if attachment.Variants == nil {
|
||||
attachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
}
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLStore) MarkAttachmentReady(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
updatedAt time.Time,
|
||||
) (domain.Attachment, error) {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
var attachment domain.Attachment
|
||||
var sha256Bytes []byte
|
||||
err := s.pool.QueryRow(
|
||||
queryContext,
|
||||
`
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, updated_at = $2
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING
|
||||
id::text, chat_id::text, owner_id::text, object_key,
|
||||
filename, content_type, size_bytes, sha256, width, height,
|
||||
status, error_code, created_at, updated_at
|
||||
`,
|
||||
attachmentID,
|
||||
updatedAt,
|
||||
).Scan(
|
||||
&attachment.ID,
|
||||
&attachment.ChatID,
|
||||
&attachment.OwnerID,
|
||||
&attachment.ObjectKey,
|
||||
&attachment.Filename,
|
||||
&attachment.ContentType,
|
||||
&attachment.SizeBytes,
|
||||
&sha256Bytes,
|
||||
&attachment.Width,
|
||||
&attachment.Height,
|
||||
&attachment.Status,
|
||||
&attachment.ErrorCode,
|
||||
&attachment.CreatedAt,
|
||||
&attachment.UpdatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return domain.Attachment{}, ErrConflict
|
||||
}
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
|
||||
}
|
||||
attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes)
|
||||
attachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLStore) MarkAttachmentFailed(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
errorCode string,
|
||||
updatedAt time.Time,
|
||||
) error {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
commandTag, err := s.pool.Exec(
|
||||
queryContext,
|
||||
`
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $2, updated_at = $3
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
`,
|
||||
attachmentID,
|
||||
errorCode,
|
||||
updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark attachment failed: %w", err)
|
||||
}
|
||||
if commandTag.RowsAffected() != 1 {
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLStore) GetAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
callerID string,
|
||||
) (domain.Attachment, error) {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
var attachment domain.Attachment
|
||||
var sha256Bytes []byte
|
||||
err := s.pool.QueryRow(
|
||||
queryContext,
|
||||
`
|
||||
SELECT
|
||||
a.id::text, a.chat_id::text, a.owner_id::text, a.object_key,
|
||||
a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.width, a.height, a.status, a.error_code,
|
||||
a.created_at, a.updated_at
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm
|
||||
ON cm.chat_id = a.chat_id AND cm.user_id = $2
|
||||
WHERE a.id = $1
|
||||
`,
|
||||
attachmentID,
|
||||
callerID,
|
||||
).Scan(
|
||||
&attachment.ID,
|
||||
&attachment.ChatID,
|
||||
&attachment.OwnerID,
|
||||
&attachment.ObjectKey,
|
||||
&attachment.Filename,
|
||||
&attachment.ContentType,
|
||||
&attachment.SizeBytes,
|
||||
&sha256Bytes,
|
||||
&attachment.Width,
|
||||
&attachment.Height,
|
||||
&attachment.Status,
|
||||
&attachment.ErrorCode,
|
||||
&attachment.CreatedAt,
|
||||
&attachment.UpdatedAt,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return domain.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("get attachment metadata: %w", err)
|
||||
}
|
||||
attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes)
|
||||
attachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLStore) ListAttachmentInventory(
|
||||
ctx context.Context,
|
||||
) ([]reconcile.AttachmentRecord, error) {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(
|
||||
queryContext,
|
||||
`
|
||||
SELECT
|
||||
a.id::text,
|
||||
a.object_key,
|
||||
a.status,
|
||||
a.updated_at,
|
||||
EXISTS (
|
||||
SELECT 1
|
||||
FROM message_attachments AS ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
) AS referenced
|
||||
FROM attachments AS a
|
||||
ORDER BY a.id
|
||||
`,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query attachment inventory: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
records := make([]reconcile.AttachmentRecord, 0)
|
||||
for rows.Next() {
|
||||
var record reconcile.AttachmentRecord
|
||||
if err = rows.Scan(
|
||||
&record.ID,
|
||||
&record.ObjectKey,
|
||||
&record.Status,
|
||||
&record.UpdatedAt,
|
||||
&record.Referenced,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan attachment inventory: %w", err)
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate attachment inventory: %w", err)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLStore) MarkAttachmentFailedIfUnreferenced(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
expectedStatus domain.AttachmentStatus,
|
||||
updatedBefore time.Time,
|
||||
errorCode string,
|
||||
updatedAt time.Time,
|
||||
) (bool, error) {
|
||||
queryContext, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
transaction, err := s.pool.BeginTx(queryContext, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin attachment reconciliation transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = transaction.Rollback(queryContext) }()
|
||||
|
||||
var lockedAttachmentID string
|
||||
err = transaction.QueryRow(
|
||||
queryContext,
|
||||
`
|
||||
SELECT id::text
|
||||
FROM attachments
|
||||
WHERE id = $1
|
||||
FOR UPDATE
|
||||
`,
|
||||
attachmentID,
|
||||
).Scan(&lockedAttachmentID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("lock attachment during reconciliation: %w", err)
|
||||
}
|
||||
|
||||
// This is deliberately a second statement. Under READ COMMITTED it gets a
|
||||
// fresh snapshot after the row lock was acquired, so a message transaction
|
||||
// that linked this attachment while we waited is visible to NOT EXISTS.
|
||||
commandTag, err := transaction.Exec(
|
||||
queryContext,
|
||||
`
|
||||
UPDATE attachments AS a
|
||||
SET status = 'failed', error_code = $4, updated_at = $5
|
||||
WHERE a.id = $1
|
||||
AND a.status = $2
|
||||
AND a.updated_at <= $3
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM message_attachments AS ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
)
|
||||
`,
|
||||
attachmentID,
|
||||
expectedStatus,
|
||||
updatedBefore,
|
||||
errorCode,
|
||||
updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("mark attachment failed during reconciliation: %w", err)
|
||||
}
|
||||
if err = transaction.Commit(queryContext); err != nil {
|
||||
return false, fmt.Errorf("commit attachment reconciliation: %w", err)
|
||||
}
|
||||
return commandTag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
var _ reconcile.MetadataStore = (*PostgreSQLStore)(nil)
|
||||
@@ -48,6 +48,59 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
|
||||
return domain.Message{}, ErrNotFound
|
||||
}
|
||||
|
||||
attachmentUUIDs := make([]uuid.UUID, len(write.AttachmentIDs))
|
||||
for index, attachmentID := range write.AttachmentIDs {
|
||||
parsedID, parseErr := uuid.Parse(attachmentID)
|
||||
if parseErr != nil {
|
||||
return domain.Message{}, fmt.Errorf("parse validated attachment id: %w", parseErr)
|
||||
}
|
||||
attachmentUUIDs[index] = parsedID
|
||||
}
|
||||
|
||||
if len(attachmentUUIDs) > 0 {
|
||||
rows, queryError := transaction.Query(
|
||||
queryContext,
|
||||
`
|
||||
SELECT id::text, status
|
||||
FROM attachments
|
||||
WHERE chat_id = $1
|
||||
AND id = ANY($2::uuid[])
|
||||
ORDER BY id
|
||||
FOR UPDATE
|
||||
`,
|
||||
write.ChatID,
|
||||
attachmentUUIDs,
|
||||
)
|
||||
if queryError != nil {
|
||||
return domain.Message{}, fmt.Errorf("lock message attachments: %w", queryError)
|
||||
}
|
||||
availableCount := 0
|
||||
readyCount := 0
|
||||
for rows.Next() {
|
||||
var attachmentID string
|
||||
var status domain.AttachmentStatus
|
||||
if queryError = rows.Scan(&attachmentID, &status); queryError != nil {
|
||||
rows.Close()
|
||||
return domain.Message{}, fmt.Errorf("scan locked message attachment: %w", queryError)
|
||||
}
|
||||
availableCount++
|
||||
if status == domain.AttachmentStatusReady {
|
||||
readyCount++
|
||||
}
|
||||
}
|
||||
queryError = rows.Err()
|
||||
rows.Close()
|
||||
if queryError != nil {
|
||||
return domain.Message{}, fmt.Errorf("iterate locked message attachments: %w", queryError)
|
||||
}
|
||||
if availableCount != len(attachmentUUIDs) {
|
||||
return domain.Message{}, ErrNotFound
|
||||
}
|
||||
if readyCount != len(attachmentUUIDs) {
|
||||
return domain.Message{}, application.ErrAttachmentNotReady
|
||||
}
|
||||
}
|
||||
|
||||
_, err = transaction.Exec(
|
||||
queryContext,
|
||||
`
|
||||
@@ -68,10 +121,11 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
|
||||
}
|
||||
|
||||
message := domain.Message{
|
||||
ID: uuid.NewString(),
|
||||
ChatID: write.ChatID,
|
||||
SenderID: write.ActorID,
|
||||
Text: write.Text,
|
||||
ID: uuid.NewString(),
|
||||
ChatID: write.ChatID,
|
||||
SenderID: write.ActorID,
|
||||
Text: write.Text,
|
||||
AttachmentIDs: append([]string(nil), write.AttachmentIDs...),
|
||||
}
|
||||
|
||||
var storedHash []byte
|
||||
@@ -141,7 +195,15 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
|
||||
chat_id::text,
|
||||
sender_id::text,
|
||||
text,
|
||||
created_at
|
||||
created_at,
|
||||
COALESCE(
|
||||
ARRAY(
|
||||
SELECT attachment_id::text
|
||||
FROM message_attachments
|
||||
WHERE message_id = messages.id
|
||||
ORDER BY position
|
||||
), ARRAY[]::text[]
|
||||
)
|
||||
FROM messages
|
||||
WHERE id = $1
|
||||
`,
|
||||
@@ -152,6 +214,7 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
)
|
||||
if err != nil {
|
||||
return domain.Message{}, fmt.Errorf("read idempotent message: %w", err)
|
||||
@@ -187,6 +250,28 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
|
||||
return domain.Message{}, fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
|
||||
if len(attachmentUUIDs) > 0 {
|
||||
_, err = transaction.Exec(
|
||||
queryContext,
|
||||
`
|
||||
INSERT INTO message_attachments (
|
||||
message_id,
|
||||
chat_id,
|
||||
attachment_id,
|
||||
position
|
||||
)
|
||||
SELECT $1, $2, attachment_id, (ordinality - 1)::smallint
|
||||
FROM unnest($3::uuid[]) WITH ORDINALITY AS input(attachment_id, ordinality)
|
||||
`,
|
||||
message.ID,
|
||||
message.ChatID,
|
||||
attachmentUUIDs,
|
||||
)
|
||||
if err != nil {
|
||||
return domain.Message{}, fmt.Errorf("link message attachments: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err = transaction.Commit(queryContext); err != nil {
|
||||
return domain.Message{}, fmt.Errorf("commit idempotent message: %w", err)
|
||||
}
|
||||
@@ -241,7 +326,15 @@ func (s *PostgreSQLStore) GetMessages(
|
||||
chat_id::text,
|
||||
sender_id::text,
|
||||
text,
|
||||
created_at
|
||||
created_at,
|
||||
COALESCE(
|
||||
ARRAY(
|
||||
SELECT attachment_id::text
|
||||
FROM message_attachments
|
||||
WHERE message_id = messages.id
|
||||
ORDER BY position
|
||||
), ARRAY[]::text[]
|
||||
)
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY
|
||||
@@ -261,7 +354,15 @@ func (s *PostgreSQLStore) GetMessages(
|
||||
chat_id::text,
|
||||
sender_id::text,
|
||||
text,
|
||||
created_at
|
||||
created_at,
|
||||
COALESCE(
|
||||
ARRAY(
|
||||
SELECT attachment_id::text
|
||||
FROM message_attachments
|
||||
WHERE message_id = messages.id
|
||||
ORDER BY position
|
||||
), ARRAY[]::text[]
|
||||
)
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND (created_at, id) > ($2, $3)
|
||||
@@ -296,6 +397,7 @@ func (s *PostgreSQLStore) GetMessages(
|
||||
&message.SenderID,
|
||||
&message.Text,
|
||||
&message.CreatedAt,
|
||||
&message.AttachmentIDs,
|
||||
); err != nil {
|
||||
return nil, false, fmt.Errorf(
|
||||
"scan message page: %w",
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
type AttachmentDownload struct {
|
||||
Body io.ReadCloser
|
||||
ContentType string
|
||||
Filename string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
func (s *Service) GetAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
actorID string,
|
||||
) (domain.Attachment, error) {
|
||||
return s.repository.GetAttachment(ctx, attachmentID, actorID)
|
||||
}
|
||||
|
||||
func (s *Service) DownloadAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
actorID string,
|
||||
variant string,
|
||||
) (AttachmentDownload, error) {
|
||||
attachment, err := s.repository.GetAttachment(ctx, attachmentID, actorID)
|
||||
if err != nil {
|
||||
return AttachmentDownload{}, err
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusReady {
|
||||
return AttachmentDownload{}, ErrAttachmentNotReady
|
||||
}
|
||||
if variant == "" {
|
||||
variant = "original"
|
||||
}
|
||||
if variant != "original" {
|
||||
return AttachmentDownload{}, domain.ErrNotFound
|
||||
}
|
||||
if s.objectStore == nil {
|
||||
return AttachmentDownload{}, ErrStorageUnavailable
|
||||
}
|
||||
|
||||
object, err := s.objectStore.Get(ctx, attachment.ObjectKey)
|
||||
if err != nil {
|
||||
return AttachmentDownload{}, fmt.Errorf("%w: get object", ErrStorageUnavailable)
|
||||
}
|
||||
if object.Body == nil || object.SizeBytes != attachment.SizeBytes {
|
||||
if object.Body != nil {
|
||||
_ = object.Body.Close()
|
||||
}
|
||||
return AttachmentDownload{}, fmt.Errorf("%w: object metadata mismatch", ErrStorageUnavailable)
|
||||
}
|
||||
return AttachmentDownload{
|
||||
Body: object.Body,
|
||||
ContentType: attachment.ContentType,
|
||||
Filename: attachment.Filename,
|
||||
SizeBytes: attachment.SizeBytes,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const attachmentPutFailedCode = "object_store_put_failed"
|
||||
const attachmentFinalizeFailedCode = "metadata_finalize_failed"
|
||||
|
||||
func (s *Service) UploadAttachment(
|
||||
ctx context.Context,
|
||||
command UploadAttachmentCommand,
|
||||
) (domain.Attachment, error) {
|
||||
if s.objectStore == nil {
|
||||
return domain.Attachment{}, ErrStorageUnavailable
|
||||
}
|
||||
if command.Body == nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: attachment body is required", ErrInvalidInput)
|
||||
}
|
||||
if err := domain.ValidateAttachmentFilename(command.Filename); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidInput, err)
|
||||
}
|
||||
if err := domain.ValidateAttachmentContentType(command.DeclaredContentType); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrUnsupportedMediaType, err)
|
||||
}
|
||||
if _, err := s.repository.GetChatByID(ctx, command.ChatID, command.ActorID); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
if err := acquireAttachmentSlot(ctx, s.attachmentUploadSlots); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: wait for upload capacity", ErrStorageUnavailable)
|
||||
}
|
||||
defer releaseAttachmentSlot(s.attachmentUploadSlots)
|
||||
|
||||
stagedFile, err := os.CreateTemp("", "mtusi-attachment-*")
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("create bounded upload staging file: %w", err)
|
||||
}
|
||||
stagedPath := stagedFile.Name()
|
||||
defer func() {
|
||||
_ = stagedFile.Close()
|
||||
_ = os.Remove(stagedPath)
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
limitedBody := io.LimitReader(command.Body, domain.MaxAttachmentSizeBytes+1)
|
||||
sizeBytes, err := io.Copy(io.MultiWriter(stagedFile, hasher), limitedBody)
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: read attachment body", ErrInvalidInput)
|
||||
}
|
||||
if sizeBytes > domain.MaxAttachmentSizeBytes {
|
||||
return domain.Attachment{}, ErrPayloadTooLarge
|
||||
}
|
||||
|
||||
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("rewind staged attachment: %w", err)
|
||||
}
|
||||
imageConfig, format, err := image.DecodeConfig(stagedFile)
|
||||
if err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: decode image header", ErrInvalidImage)
|
||||
}
|
||||
actualContentType, supported := contentTypeForImageFormat(format)
|
||||
if !supported || actualContentType != command.DeclaredContentType {
|
||||
return domain.Attachment{}, ErrUnsupportedMediaType
|
||||
}
|
||||
if imageConfig.Width < 1 || imageConfig.Height < 1 {
|
||||
return domain.Attachment{}, ErrInvalidImage
|
||||
}
|
||||
if int64(imageConfig.Width) > int64(domain.MaxAttachmentPixels)/int64(imageConfig.Height) {
|
||||
return domain.Attachment{}, ErrPayloadTooLarge
|
||||
}
|
||||
|
||||
sha256Hex := hex.EncodeToString(hasher.Sum(nil))
|
||||
if err = domain.ValidateReadyAttachmentMetadata(
|
||||
command.Filename,
|
||||
actualContentType,
|
||||
sizeBytes,
|
||||
sha256Hex,
|
||||
imageConfig.Width,
|
||||
imageConfig.Height,
|
||||
); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidImage, err)
|
||||
}
|
||||
|
||||
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
|
||||
return domain.Attachment{}, fmt.Errorf("rewind staged attachment for validation: %w", err)
|
||||
}
|
||||
if err = s.validateCompleteImage(ctx, stagedFile); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return domain.Attachment{}, fmt.Errorf("%w: wait for image decoder", ErrStorageUnavailable)
|
||||
}
|
||||
return domain.Attachment{}, fmt.Errorf("%w: decode complete image", ErrInvalidImage)
|
||||
}
|
||||
|
||||
now := s.now().UTC()
|
||||
attachmentID := uuid.NewString()
|
||||
attachment := domain.Attachment{
|
||||
ID: attachmentID,
|
||||
ChatID: command.ChatID,
|
||||
OwnerID: command.ActorID,
|
||||
Filename: command.Filename,
|
||||
ObjectKey: "attachments/" + command.ChatID + "/" + attachmentID + "/original",
|
||||
ContentType: actualContentType,
|
||||
SizeBytes: sizeBytes,
|
||||
SHA256: sha256Hex,
|
||||
Width: imageConfig.Width,
|
||||
Height: imageConfig.Height,
|
||||
Status: domain.AttachmentStatusPending,
|
||||
Variants: make([]domain.AttachmentVariant, 0),
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
attachment, err = s.repository.CreateAttachment(ctx, attachment)
|
||||
if err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
|
||||
s.cleanupFailedAttachment(ctx, attachment, attachmentFinalizeFailedCode, false)
|
||||
return domain.Attachment{}, fmt.Errorf("rewind staged attachment for storage: %w", err)
|
||||
}
|
||||
err = s.objectStore.Put(ctx, ObjectPut{
|
||||
Key: attachment.ObjectKey,
|
||||
ContentType: attachment.ContentType,
|
||||
SizeBytes: attachment.SizeBytes,
|
||||
Body: stagedFile,
|
||||
})
|
||||
if err != nil {
|
||||
s.cleanupFailedAttachment(ctx, attachment, attachmentPutFailedCode, true)
|
||||
return domain.Attachment{}, fmt.Errorf("%w: put object", ErrStorageUnavailable)
|
||||
}
|
||||
|
||||
readyAttachment, err := s.repository.MarkAttachmentReady(ctx, attachment.ID, s.now().UTC())
|
||||
if err != nil {
|
||||
// The UPDATE may already have committed even when its response was lost.
|
||||
// Deleting the object here could turn durable ready metadata into a
|
||||
// permanently broken attachment. A best-effort read may recover a 201;
|
||||
// otherwise both sides are preserved for the grace-period reconciler.
|
||||
if resolved, ok := s.resolveAmbiguousAttachmentFinalize(ctx, attachment); ok {
|
||||
return resolved, nil
|
||||
}
|
||||
return domain.Attachment{}, fmt.Errorf("finalize attachment metadata: %w", err)
|
||||
}
|
||||
if readyAttachment.Variants == nil {
|
||||
readyAttachment.Variants = make([]domain.AttachmentVariant, 0)
|
||||
}
|
||||
return readyAttachment, nil
|
||||
}
|
||||
|
||||
func acquireAttachmentSlot(ctx context.Context, slots chan struct{}) error {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func releaseAttachmentSlot(slots chan struct{}) {
|
||||
<-slots
|
||||
}
|
||||
|
||||
func (s *Service) validateCompleteImage(ctx context.Context, body io.Reader) error {
|
||||
if err := acquireAttachmentSlot(ctx, s.attachmentDecodeSlots); err != nil {
|
||||
return err
|
||||
}
|
||||
defer releaseAttachmentSlot(s.attachmentDecodeSlots)
|
||||
_, _, err := image.Decode(body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) resolveAmbiguousAttachmentFinalize(
|
||||
requestContext context.Context,
|
||||
attachment domain.Attachment,
|
||||
) (domain.Attachment, bool) {
|
||||
resolveContext, cancel := context.WithTimeout(
|
||||
context.WithoutCancel(requestContext),
|
||||
s.attachmentCleanupTimeout,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
stored, err := s.repository.GetAttachment(
|
||||
resolveContext,
|
||||
attachment.ID,
|
||||
attachment.OwnerID,
|
||||
)
|
||||
if err != nil || stored.Status != domain.AttachmentStatusReady {
|
||||
return domain.Attachment{}, false
|
||||
}
|
||||
if stored.Variants == nil {
|
||||
stored.Variants = make([]domain.AttachmentVariant, 0)
|
||||
}
|
||||
return stored, true
|
||||
}
|
||||
|
||||
func contentTypeForImageFormat(format string) (string, bool) {
|
||||
switch format {
|
||||
case "png":
|
||||
return "image/png", true
|
||||
case "jpeg":
|
||||
return "image/jpeg", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) cleanupFailedAttachment(
|
||||
requestContext context.Context,
|
||||
attachment domain.Attachment,
|
||||
errorCode string,
|
||||
deleteObject bool,
|
||||
) {
|
||||
metadataContext, metadataCancel := context.WithTimeout(
|
||||
context.WithoutCancel(requestContext),
|
||||
s.attachmentCleanupTimeout,
|
||||
)
|
||||
_ = s.repository.MarkAttachmentFailed(
|
||||
metadataContext,
|
||||
attachment.ID,
|
||||
errorCode,
|
||||
s.now().UTC(),
|
||||
)
|
||||
metadataCancel()
|
||||
|
||||
if !deleteObject {
|
||||
return
|
||||
}
|
||||
deleteContext, deleteCancel := context.WithTimeout(
|
||||
context.WithoutCancel(requestContext),
|
||||
s.attachmentCleanupTimeout,
|
||||
)
|
||||
defer deleteCancel()
|
||||
_ = s.objectStore.Delete(deleteContext, attachment.ObjectKey)
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
package application_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/adapter/memory"
|
||||
. "mtusi2026lab4/internal/application"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
type fakeObjectStore struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
putError error
|
||||
blockDelete bool
|
||||
putCalls int
|
||||
deleteCalls int
|
||||
lastPutKey string
|
||||
}
|
||||
|
||||
type alwaysFailReader struct{}
|
||||
|
||||
func (alwaysFailReader) Read([]byte) (int, error) {
|
||||
return 0, errors.New("injected interrupted upload")
|
||||
}
|
||||
|
||||
type finalizeFailureRepository struct {
|
||||
Repository
|
||||
}
|
||||
|
||||
func (repository *finalizeFailureRepository) MarkAttachmentReady(
|
||||
context.Context,
|
||||
string,
|
||||
time.Time,
|
||||
) (domain.Attachment, error) {
|
||||
return domain.Attachment{}, errors.New("injected metadata finalize failure")
|
||||
}
|
||||
|
||||
type commitThenDisconnectRepository struct {
|
||||
Repository
|
||||
hideReadback bool
|
||||
}
|
||||
|
||||
func (repository *commitThenDisconnectRepository) MarkAttachmentReady(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
updatedAt time.Time,
|
||||
) (domain.Attachment, error) {
|
||||
if _, err := repository.Repository.MarkAttachmentReady(ctx, attachmentID, updatedAt); err != nil {
|
||||
return domain.Attachment{}, err
|
||||
}
|
||||
return domain.Attachment{}, errors.New("injected connection loss after commit")
|
||||
}
|
||||
|
||||
func (repository *commitThenDisconnectRepository) GetAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
actorID string,
|
||||
) (domain.Attachment, error) {
|
||||
if repository.hideReadback {
|
||||
return domain.Attachment{}, errors.New("injected database outage during readback")
|
||||
}
|
||||
return repository.Repository.GetAttachment(ctx, attachmentID, actorID)
|
||||
}
|
||||
|
||||
func newFakeObjectStore() *fakeObjectStore {
|
||||
return &fakeObjectStore{objects: make(map[string][]byte)}
|
||||
}
|
||||
|
||||
func (store *fakeObjectStore) Put(_ context.Context, object ObjectPut) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
store.putCalls++
|
||||
store.lastPutKey = object.Key
|
||||
if store.putError != nil {
|
||||
return store.putError
|
||||
}
|
||||
payload, err := io.ReadAll(object.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(len(payload)) != object.SizeBytes {
|
||||
return errors.New("declared object size differs")
|
||||
}
|
||||
store.objects[object.Key] = payload
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *fakeObjectStore) Get(_ context.Context, key string) (ObjectContent, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
payload, exists := store.objects[key]
|
||||
if !exists {
|
||||
return ObjectContent{}, errors.New("object not found")
|
||||
}
|
||||
copyOfPayload := append([]byte(nil), payload...)
|
||||
return ObjectContent{
|
||||
Body: io.NopCloser(bytes.NewReader(copyOfPayload)),
|
||||
SizeBytes: int64(len(copyOfPayload)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store *fakeObjectStore) Delete(ctx context.Context, key string) error {
|
||||
store.mu.Lock()
|
||||
store.deleteCalls++
|
||||
blockDelete := store.blockDelete
|
||||
store.mu.Unlock()
|
||||
if blockDelete {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
delete(store.objects, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func newAttachmentService(
|
||||
t *testing.T,
|
||||
objectStore ObjectStore,
|
||||
) (*Service, *memory.MemoryStore, string, string) {
|
||||
t.Helper()
|
||||
legacyStore := memory.NewMemoryStore()
|
||||
repository := memory.NewMemoryRepository(legacyStore)
|
||||
user, err := repository.CreateUser(context.Background(), "uploader", "Uploader", "hash")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chat, err := repository.CreateChat(context.Background(), "Images", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
codec, err := NewCursorCodec("attachment-test-key-with-more-than-thirty-two-bytes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return service, legacyStore, user.ID, chat.ID
|
||||
}
|
||||
|
||||
func encodeTestImage(t *testing.T, contentType string) []byte {
|
||||
t.Helper()
|
||||
var payload bytes.Buffer
|
||||
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
|
||||
img.Set(0, 0, color.RGBA{R: 255, A: 255})
|
||||
var err error
|
||||
if contentType == "image/png" {
|
||||
err = png.Encode(&payload, img)
|
||||
} else {
|
||||
err = jpeg.Encode(&payload, img, nil)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return payload.Bytes()
|
||||
}
|
||||
|
||||
func corruptPNGAfterPrefix(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
img := image.NewNRGBA(image.Rect(0, 0, 512, 512))
|
||||
state := uint32(0x12345678)
|
||||
for index := range img.Pix {
|
||||
state ^= state << 13
|
||||
state ^= state >> 17
|
||||
state ^= state << 5
|
||||
img.Pix[index] = byte(state)
|
||||
}
|
||||
var payload bytes.Buffer
|
||||
if err := png.Encode(&payload, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
corrupted := payload.Bytes()
|
||||
if len(corrupted) <= 70_000 {
|
||||
t.Fatalf("test PNG is unexpectedly small: %d", len(corrupted))
|
||||
}
|
||||
corrupted[70_000] ^= 0xff
|
||||
return corrupted
|
||||
}
|
||||
|
||||
func jpegWithoutScanData(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
payload := encodeTestImage(t, "image/jpeg")
|
||||
sos := bytes.Index(payload, []byte{0xff, 0xda})
|
||||
if sos < 0 || sos+4 > len(payload) {
|
||||
t.Fatal("encoded JPEG has no SOS marker")
|
||||
}
|
||||
segmentLength := int(binary.BigEndian.Uint16(payload[sos+2 : sos+4]))
|
||||
endOfHeader := sos + 2 + segmentLength
|
||||
if endOfHeader > len(payload) {
|
||||
t.Fatal("invalid SOS segment length in test JPEG")
|
||||
}
|
||||
return append([]byte(nil), payload[:endOfHeader]...)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentStoresValidatedImage(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
payload := encodeTestImage(t, "image/png")
|
||||
|
||||
attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID,
|
||||
ChatID: chatID,
|
||||
Filename: "кот.png",
|
||||
DeclaredContentType: "image/png",
|
||||
Body: bytes.NewReader(payload),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusReady {
|
||||
t.Fatalf("status = %q, want ready", attachment.Status)
|
||||
}
|
||||
if attachment.ObjectKey == "" || strings.Contains(attachment.ObjectKey, attachment.Filename) {
|
||||
t.Fatalf("object key must be server-generated: %q", attachment.ObjectKey)
|
||||
}
|
||||
if attachment.Width != 2 || attachment.Height != 2 || len(attachment.SHA256) != 64 {
|
||||
t.Fatalf("unexpected image metadata: %+v", attachment)
|
||||
}
|
||||
if !bytes.Equal(objectStore.objects[attachment.ObjectKey], payload) {
|
||||
t.Fatal("stored object differs from uploaded bytes")
|
||||
}
|
||||
stored, err := metadataStore.GetAttachmentByID(attachment.ID)
|
||||
if err != nil || stored.Status != domain.AttachmentStatusReady {
|
||||
t.Fatalf("ready metadata was not persisted: %+v, %v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentRejectsInvalidInputBeforeStorage(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
filename string
|
||||
declaredType string
|
||||
payload []byte
|
||||
wantError error
|
||||
}{
|
||||
{name: "unsafe filename", filename: "../image.png", declaredType: "image/png", payload: encodeTestImage(t, "image/png"), wantError: ErrInvalidInput},
|
||||
{name: "unsupported declared type", filename: "image.gif", declaredType: "image/gif", payload: []byte("GIF89a"), wantError: ErrUnsupportedMediaType},
|
||||
{name: "MIME substitution", filename: "image.jpg", declaredType: "image/jpeg", payload: encodeTestImage(t, "image/png"), wantError: ErrUnsupportedMediaType},
|
||||
{name: "corrupt image", filename: "image.png", declaredType: "image/png", payload: []byte("not a png"), wantError: ErrInvalidImage},
|
||||
{name: "PNG header without image data", filename: "truncated.png", declaredType: "image/png", payload: pngConfigurationOnly(1, 1), wantError: ErrInvalidImage},
|
||||
{name: "PNG corruption after 64 KiB", filename: "corrupted.png", declaredType: "image/png", payload: corruptPNGAfterPrefix(t), wantError: ErrInvalidImage},
|
||||
{name: "JPEG without scan data", filename: "truncated.jpg", declaredType: "image/jpeg", payload: jpegWithoutScanData(t), wantError: ErrInvalidImage},
|
||||
{name: "too large", filename: "image.png", declaredType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantError: ErrPayloadTooLarge},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
service, _, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID,
|
||||
ChatID: chatID,
|
||||
Filename: testCase.filename,
|
||||
DeclaredContentType: testCase.declaredType,
|
||||
Body: bytes.NewReader(testCase.payload),
|
||||
})
|
||||
if !errors.Is(err, testCase.wantError) {
|
||||
t.Fatalf("error = %v, want %v", err, testCase.wantError)
|
||||
}
|
||||
if objectStore.putCalls != 0 {
|
||||
t.Fatalf("invalid upload reached object storage %d times", objectStore.putCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentCompensatesObjectStoreFailure(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
objectStore.putError = errors.New("injected put failure")
|
||||
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
|
||||
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID,
|
||||
ChatID: chatID,
|
||||
Filename: "image.jpg",
|
||||
DeclaredContentType: "image/jpeg",
|
||||
Body: bytes.NewReader(encodeTestImage(t, "image/jpeg")),
|
||||
})
|
||||
if !errors.Is(err, ErrStorageUnavailable) {
|
||||
t.Fatalf("error = %v, want ErrStorageUnavailable", err)
|
||||
}
|
||||
if objectStore.deleteCalls != 1 {
|
||||
t.Fatalf("compensating deletes = %d, want 1", objectStore.deleteCalls)
|
||||
}
|
||||
|
||||
objectStore.mu.Lock()
|
||||
keyParts := strings.Split(objectStore.lastPutKey, "/")
|
||||
objectStore.mu.Unlock()
|
||||
if len(keyParts) != 4 {
|
||||
t.Fatalf("unexpected generated object key: %q", objectStore.lastPutKey)
|
||||
}
|
||||
failedAttachment, lookupErr := metadataStore.GetAttachmentByID(keyParts[2])
|
||||
if lookupErr != nil {
|
||||
t.Fatal(lookupErr)
|
||||
}
|
||||
if failedAttachment.Status != domain.AttachmentStatusFailed || failedAttachment.ErrorCode == nil {
|
||||
t.Fatalf("failed upload metadata was not finalized: %+v", failedAttachment)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentMarksFailedBeforeBlockedObjectCleanup(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
objectStore.putError = errors.New("injected put failure")
|
||||
objectStore.blockDelete = true
|
||||
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
|
||||
startedAt := time.Now()
|
||||
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID, ChatID: chatID, Filename: "blocked-cleanup.png",
|
||||
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
|
||||
})
|
||||
if !errors.Is(err, ErrStorageUnavailable) {
|
||||
t.Fatalf("error = %v, want ErrStorageUnavailable", err)
|
||||
}
|
||||
if elapsed := time.Since(startedAt); elapsed > 2*time.Second {
|
||||
t.Fatalf("bounded cleanup took %s", elapsed)
|
||||
}
|
||||
keyParts := strings.Split(objectStore.lastPutKey, "/")
|
||||
failed, lookupErr := metadataStore.GetAttachmentByID(keyParts[2])
|
||||
if lookupErr != nil {
|
||||
t.Fatal(lookupErr)
|
||||
}
|
||||
if failed.Status != domain.AttachmentStatusFailed || failed.ErrorCode == nil {
|
||||
t.Fatalf("blocked object cleanup consumed DB cleanup context: %+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentRejectsInterruptedBodyBeforeMetadata(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
payload := encodeTestImage(t, "image/png")
|
||||
|
||||
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID,
|
||||
ChatID: chatID,
|
||||
Filename: "interrupted.png",
|
||||
DeclaredContentType: "image/png",
|
||||
Body: io.MultiReader(bytes.NewReader(payload[:8]), alwaysFailReader{}),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidInput) {
|
||||
t.Fatalf("error = %v, want ErrInvalidInput", err)
|
||||
}
|
||||
if objectStore.putCalls != 0 {
|
||||
t.Fatal("interrupted upload reached object storage")
|
||||
}
|
||||
if _, lookupErr := metadataStore.GetAttachmentByID("missing"); !errors.Is(lookupErr, domain.ErrNotFound) {
|
||||
t.Fatalf("unexpected metadata lookup result: %v", lookupErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentPreservesObjectOnMetadataFinalizeFailure(t *testing.T) {
|
||||
legacyStore := memory.NewMemoryStore()
|
||||
repository := memory.NewMemoryRepository(legacyStore)
|
||||
user, err := repository.CreateUser(context.Background(), "finalize", "Finalize", "hash")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chat, err := repository.CreateChat(context.Background(), "Finalize", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
codec, err := NewCursorCodec("finalize-test-key-with-more-than-thirty-two-bytes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objectStore := newFakeObjectStore()
|
||||
service, err := NewService(
|
||||
&finalizeFailureRepository{Repository: repository},
|
||||
codec,
|
||||
time.Hour,
|
||||
WithObjectStore(objectStore),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: user.ID,
|
||||
ChatID: chat.ID,
|
||||
Filename: "finalize.png",
|
||||
DeclaredContentType: "image/png",
|
||||
Body: bytes.NewReader(encodeTestImage(t, "image/png")),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("metadata finalize failure was hidden")
|
||||
}
|
||||
if objectStore.deleteCalls != 0 || len(objectStore.objects) != 1 {
|
||||
t.Fatalf("ambiguous object must be preserved: deletes=%d objects=%d", objectStore.deleteCalls, len(objectStore.objects))
|
||||
}
|
||||
keyParts := strings.Split(objectStore.lastPutKey, "/")
|
||||
pending, lookupErr := legacyStore.GetAttachmentByID(keyParts[2])
|
||||
if lookupErr != nil {
|
||||
t.Fatal(lookupErr)
|
||||
}
|
||||
if pending.Status != domain.AttachmentStatusPending || pending.ErrorCode != nil {
|
||||
t.Fatalf("ambiguous metadata must remain reconcilable: %+v", pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentRecoversWhenCommitResponseIsLost(t *testing.T) {
|
||||
legacyStore := memory.NewMemoryStore()
|
||||
baseRepository := memory.NewMemoryRepository(legacyStore)
|
||||
user, err := baseRepository.CreateUser(context.Background(), "commit-lost", "Commit Lost", "hash")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chat, err := baseRepository.CreateChat(context.Background(), "Commit Lost", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
codec, err := NewCursorCodec("commit-lost-test-key-with-more-than-thirty-two-bytes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objectStore := newFakeObjectStore()
|
||||
repository := &commitThenDisconnectRepository{Repository: baseRepository}
|
||||
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png",
|
||||
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("committed ready attachment was not recovered: %v", err)
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 {
|
||||
t.Fatalf("lost commit response damaged attachment: attachment=%+v deletes=%d objects=%d",
|
||||
attachment, objectStore.deleteCalls, len(objectStore.objects))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentNeverDeletesObjectAfterAmbiguousCommit(t *testing.T) {
|
||||
legacyStore := memory.NewMemoryStore()
|
||||
baseRepository := memory.NewMemoryRepository(legacyStore)
|
||||
user, err := baseRepository.CreateUser(context.Background(), "commit-outage", "Commit Outage", "hash")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chat, err := baseRepository.CreateChat(context.Background(), "Commit Outage", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
codec, err := NewCursorCodec("commit-outage-test-key-with-more-than-thirty-two-bytes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objectStore := newFakeObjectStore()
|
||||
repository := &commitThenDisconnectRepository{Repository: baseRepository, hideReadback: true}
|
||||
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png",
|
||||
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("database outage after commit was hidden")
|
||||
}
|
||||
keyParts := strings.Split(objectStore.lastPutKey, "/")
|
||||
stored, lookupErr := legacyStore.GetAttachmentByID(keyParts[2])
|
||||
if lookupErr != nil {
|
||||
t.Fatal(lookupErr)
|
||||
}
|
||||
if stored.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 {
|
||||
t.Fatalf("committed ready metadata lost its original: metadata=%+v deletes=%d objects=%d",
|
||||
stored, objectStore.deleteCalls, len(objectStore.objects))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentRejectsPixelLimitBeforeFullDecode(t *testing.T) {
|
||||
objectStore := newFakeObjectStore()
|
||||
service, _, actorID, chatID := newAttachmentService(t, objectStore)
|
||||
|
||||
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
|
||||
ActorID: actorID,
|
||||
ChatID: chatID,
|
||||
Filename: "huge.png",
|
||||
DeclaredContentType: "image/png",
|
||||
Body: bytes.NewReader(pngConfigurationOnly(10_000, 4_001)),
|
||||
})
|
||||
if !errors.Is(err, ErrPayloadTooLarge) {
|
||||
t.Fatalf("error = %v, want ErrPayloadTooLarge", err)
|
||||
}
|
||||
if objectStore.putCalls != 0 {
|
||||
t.Fatalf("oversized decoded image reached object storage %d times", objectStore.putCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func pngConfigurationOnly(width uint32, height uint32) []byte {
|
||||
payload := make([]byte, 33)
|
||||
copy(payload[0:8], []byte{137, 80, 78, 71, 13, 10, 26, 10})
|
||||
binary.BigEndian.PutUint32(payload[8:12], 13)
|
||||
copy(payload[12:16], "IHDR")
|
||||
binary.BigEndian.PutUint32(payload[16:20], width)
|
||||
binary.BigEndian.PutUint32(payload[20:24], height)
|
||||
payload[24] = 8
|
||||
payload[25] = 6
|
||||
payload[26] = 0
|
||||
payload[27] = 0
|
||||
payload[28] = 0
|
||||
binary.BigEndian.PutUint32(payload[29:33], crc32.ChecksumIEEE(payload[12:29]))
|
||||
return payload
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package application
|
||||
|
||||
import "io"
|
||||
|
||||
type CreateUserCommand struct {
|
||||
Username string
|
||||
DisplayName string
|
||||
@@ -21,7 +23,16 @@ type SendMessageCommand struct {
|
||||
ActorID string
|
||||
ChatID string
|
||||
Text string
|
||||
AttachmentIDs []string
|
||||
Method string
|
||||
CanonicalPath string
|
||||
IdempotencyKey string
|
||||
}
|
||||
|
||||
type UploadAttachmentCommand struct {
|
||||
ActorID string
|
||||
ChatID string
|
||||
Filename string
|
||||
DeclaredContentType string
|
||||
Body io.Reader
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -47,11 +49,13 @@ func TestCursorCodecRejectsTamperingAndWrongScope(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
replacement := "A"
|
||||
if token[len(token)-1:] == replacement {
|
||||
replacement = "B"
|
||||
parts := strings.Split(token, ".")
|
||||
signature, err := base64.RawURLEncoding.DecodeString(parts[1])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tampered := token[:len(token)-1] + replacement
|
||||
signature[0] ^= 1
|
||||
tampered := parts[0] + "." + base64.RawURLEncoding.EncodeToString(signature)
|
||||
if _, err = codec.DecodeChat(tampered, "9fa3f49c-e62e-4209-8414-d6d5f69805bf"); !errors.Is(err, ErrInvalidCursor) {
|
||||
t.Fatalf("tampered cursor: got %v, want ErrInvalidCursor", err)
|
||||
}
|
||||
@@ -68,3 +72,25 @@ func TestCursorCodecRequiresStrongSharedKey(t *testing.T) {
|
||||
t.Fatal("expected short cursor signing key to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCursorCodecRejectsSignedInvalidFieldTypesBeforeSQL(t *testing.T) {
|
||||
codec, err := NewCursorCodec(cursorTestKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chatID := "9fa3f49c-e62e-4209-8414-d6d5f69805bf"
|
||||
validMessageID := "42eea32e-d9b5-4bc6-b4fc-94e9b0477adf"
|
||||
testCases := []cursorPayload{
|
||||
{Version: cursorVersion, Kind: "message", CreatedAt: "not-a-time", ID: validMessageID, ScopeID: chatID},
|
||||
{Version: cursorVersion, Kind: "message", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), ID: "not-a-uuid", ScopeID: chatID},
|
||||
}
|
||||
for _, payload := range testCases {
|
||||
token, encodeErr := codec.encode(payload)
|
||||
if encodeErr != nil {
|
||||
t.Fatal(encodeErr)
|
||||
}
|
||||
if _, decodeErr := codec.DecodeMessage(token, chatID); !errors.Is(decodeErr, ErrInvalidCursor) {
|
||||
t.Fatalf("invalid signed cursor returned %v, want ErrInvalidCursor", decodeErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,14 @@ package application
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrInvalidCredentials = errors.New("invalid credentials")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrCredentialState = errors.New("invalid stored credential state")
|
||||
ErrIdempotencyConflict = errors.New("idempotency key is already used for another request")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
ErrInvalidCredentials = errors.New("invalid credentials")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrCredentialState = errors.New("invalid stored credential state")
|
||||
ErrIdempotencyConflict = errors.New("idempotency key is already used for another request")
|
||||
ErrAttachmentNotReady = errors.New("attachment is not ready")
|
||||
ErrPayloadTooLarge = errors.New("payload too large")
|
||||
ErrUnsupportedMediaType = errors.New("unsupported media type")
|
||||
ErrInvalidImage = errors.New("invalid image")
|
||||
ErrStorageUnavailable = errors.New("object storage unavailable")
|
||||
)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type canonicalMessageRequest struct {
|
||||
@@ -15,13 +17,21 @@ type canonicalMessageRequest struct {
|
||||
}
|
||||
|
||||
func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) (domain.Message, error) {
|
||||
if err := domain.ValidateMessageText(command.Text); err != nil {
|
||||
attachmentIDs := append([]string(nil), command.AttachmentIDs...)
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = make([]string, 0)
|
||||
}
|
||||
|
||||
if err := domain.ValidateMessageInput(command.Text, attachmentIDs); err != nil {
|
||||
return domain.Message{}, fmt.Errorf("%w: %v", ErrInvalidInput, err)
|
||||
}
|
||||
for index, attachmentID := range attachmentIDs {
|
||||
attachmentIDs[index] = uuid.MustParse(attachmentID).String()
|
||||
}
|
||||
|
||||
canonicalBody, err := json.Marshal(canonicalMessageRequest{
|
||||
Text: command.Text,
|
||||
AttachmentIDs: make([]string, 0),
|
||||
AttachmentIDs: attachmentIDs,
|
||||
})
|
||||
if err != nil {
|
||||
return domain.Message{}, fmt.Errorf("canonicalize message request: %w", err)
|
||||
@@ -31,6 +41,7 @@ func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) (
|
||||
ActorID: command.ActorID,
|
||||
ChatID: command.ChatID,
|
||||
Text: command.Text,
|
||||
AttachmentIDs: attachmentIDs,
|
||||
Method: command.Method,
|
||||
CanonicalPath: command.CanonicalPath,
|
||||
Key: command.IdempotencyKey,
|
||||
|
||||
@@ -2,6 +2,7 @@ package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
@@ -28,6 +29,7 @@ type IdempotentMessageWrite struct {
|
||||
ActorID string
|
||||
ChatID string
|
||||
Text string
|
||||
AttachmentIDs []string
|
||||
Method string
|
||||
CanonicalPath string
|
||||
Key string
|
||||
@@ -40,9 +42,35 @@ type MessageRepository interface {
|
||||
GetMessages(ctx context.Context, chatID, callerID string, limit int, cursor *domain.MessageCursor) ([]domain.Message, bool, error)
|
||||
}
|
||||
|
||||
type AttachmentRepository interface {
|
||||
CreateAttachment(ctx context.Context, attachment domain.Attachment) (domain.Attachment, error)
|
||||
MarkAttachmentReady(ctx context.Context, attachmentID string, updatedAt time.Time) (domain.Attachment, error)
|
||||
MarkAttachmentFailed(ctx context.Context, attachmentID, errorCode string, updatedAt time.Time) error
|
||||
GetAttachment(ctx context.Context, attachmentID, callerID string) (domain.Attachment, error)
|
||||
}
|
||||
|
||||
type Repository interface {
|
||||
UserRepository
|
||||
SessionRepository
|
||||
ChatRepository
|
||||
MessageRepository
|
||||
AttachmentRepository
|
||||
}
|
||||
|
||||
type ObjectPut struct {
|
||||
Key string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Body io.Reader
|
||||
}
|
||||
|
||||
type ObjectContent struct {
|
||||
Body io.ReadCloser
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Put(ctx context.Context, object ObjectPut) error
|
||||
Get(ctx context.Context, key string) (ObjectContent, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
@@ -6,15 +6,22 @@ import (
|
||||
)
|
||||
|
||||
const MinimumIdempotencyTTL = 24 * time.Hour
|
||||
const defaultAttachmentCleanupTimeout = time.Second
|
||||
const defaultAttachmentUploadConcurrency = 4
|
||||
const defaultAttachmentDecodeConcurrency = 1
|
||||
|
||||
type ServiceOption func(*Service) error
|
||||
|
||||
type Service struct {
|
||||
repository Repository
|
||||
cursors *CursorCodec
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
now func() time.Time
|
||||
repository Repository
|
||||
cursors *CursorCodec
|
||||
sessionTTL time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
now func() time.Time
|
||||
objectStore ObjectStore
|
||||
attachmentCleanupTimeout time.Duration
|
||||
attachmentUploadSlots chan struct{}
|
||||
attachmentDecodeSlots chan struct{}
|
||||
}
|
||||
|
||||
func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Duration, options ...ServiceOption) (*Service, error) {
|
||||
@@ -28,11 +35,14 @@ func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Dur
|
||||
return nil, fmt.Errorf("session TTL must be positive")
|
||||
}
|
||||
service := &Service{
|
||||
repository: repository,
|
||||
cursors: cursors,
|
||||
sessionTTL: sessionTTL,
|
||||
idempotencyTTL: MinimumIdempotencyTTL,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
repository: repository,
|
||||
cursors: cursors,
|
||||
sessionTTL: sessionTTL,
|
||||
idempotencyTTL: MinimumIdempotencyTTL,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
attachmentCleanupTimeout: defaultAttachmentCleanupTimeout,
|
||||
attachmentUploadSlots: make(chan struct{}, defaultAttachmentUploadConcurrency),
|
||||
attachmentDecodeSlots: make(chan struct{}, defaultAttachmentDecodeConcurrency),
|
||||
}
|
||||
for _, option := range options {
|
||||
if err := option(service); err != nil {
|
||||
@@ -51,3 +61,13 @@ func WithIdempotencyTTL(ttl time.Duration) ServiceOption {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithObjectStore(objectStore ObjectStore) ServiceOption {
|
||||
return func(service *Service) error {
|
||||
if objectStore == nil {
|
||||
return fmt.Errorf("object store is required")
|
||||
}
|
||||
service.objectStore = objectStore
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,13 @@ type Config struct {
|
||||
RedisOperationTimeout time.Duration
|
||||
RedisPoolSize int
|
||||
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKeyID string
|
||||
S3SecretAccessKey string
|
||||
S3OperationTimeout time.Duration
|
||||
|
||||
ReadinessTimeout time.Duration
|
||||
ShutdownDrainDelay time.Duration
|
||||
ShutdownTimeout time.Duration
|
||||
@@ -82,6 +89,24 @@ func Load() (Config, error) {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
s3AccessKeyID, err := requiredValueOrFile(
|
||||
"S3_ACCESS_KEY_ID",
|
||||
"S3_ACCESS_KEY_ID_FILE",
|
||||
)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3SecretAccessKey, err := requiredValueOrFile(
|
||||
"S3_SECRET_ACCESS_KEY",
|
||||
"S3_SECRET_ACCESS_KEY_FILE",
|
||||
)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
s3OperationTimeout, err := durationFromEnv("S3_OPERATION_TIMEOUT", 5*time.Second)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
databaseMaxConns, err := int32FromEnv("DB_MAX_CONNS", 10, 1)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
@@ -223,6 +248,12 @@ func Load() (Config, error) {
|
||||
RedisOperationTimeout: redisOperationTimeout,
|
||||
RedisPoolSize: int(redisPoolSize),
|
||||
|
||||
S3Endpoint: envOrDefault("S3_ENDPOINT", "http://minio:9000"),
|
||||
S3Region: envOrDefault("S3_REGION", "us-east-1"),
|
||||
S3Bucket: envOrDefault("S3_BUCKET", "messenger-attachments"),
|
||||
S3AccessKeyID: s3AccessKeyID,
|
||||
S3SecretAccessKey: s3SecretAccessKey,
|
||||
S3OperationTimeout: s3OperationTimeout,
|
||||
ReadinessTimeout: readinessTimeout,
|
||||
ShutdownDrainDelay: shutdownDrainDelay,
|
||||
ShutdownTimeout: shutdownTimeout,
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxAttachmentFilenameCodePoints = 255
|
||||
MaxAttachmentSizeBytes = 10 * 1024 * 1024
|
||||
MaxAttachmentPixels = 40_000_000
|
||||
)
|
||||
|
||||
type AttachmentVariant struct {
|
||||
Name string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
type AttachmentStatus string
|
||||
|
||||
const (
|
||||
AttachmentStatusPending AttachmentStatus = "pending"
|
||||
AttachmentStatusReady AttachmentStatus = "ready"
|
||||
AttachmentStatusFailed AttachmentStatus = "failed"
|
||||
)
|
||||
|
||||
// Attachment contains metadata only. ObjectKey points to bytes in the
|
||||
// private object store and must never be derived from Filename.
|
||||
type Attachment struct {
|
||||
ID string
|
||||
ChatID string
|
||||
OwnerID string
|
||||
Filename string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
Width int
|
||||
Height int
|
||||
Status AttachmentStatus
|
||||
Variants []AttachmentVariant
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func ValidateAttachmentFilename(filename string) error {
|
||||
if !utf8.ValidString(filename) {
|
||||
return fmt.Errorf("filename must be valid UTF-8")
|
||||
}
|
||||
filenameLength := utf8.RuneCountInString(filename)
|
||||
if filenameLength < 1 || filenameLength > MaxAttachmentFilenameCodePoints {
|
||||
return fmt.Errorf("filename length must be between 1 and %d code points", MaxAttachmentFilenameCodePoints)
|
||||
}
|
||||
if strings.TrimSpace(filename) == "" || filename == "." || filename == ".." {
|
||||
return fmt.Errorf("filename is unsafe")
|
||||
}
|
||||
if strings.ContainsAny(filename, `/\\`) {
|
||||
return fmt.Errorf("filename must not contain path separators")
|
||||
}
|
||||
for _, character := range filename {
|
||||
if unicode.IsControl(character) {
|
||||
return fmt.Errorf("filename must not contain control characters")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateAttachmentContentType(contentType string) error {
|
||||
if contentType != "image/png" && contentType != "image/jpeg" {
|
||||
return fmt.Errorf("content type must be image/png or image/jpeg")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateReadyAttachmentMetadata checks the bounded metadata produced after
|
||||
// the upload stream has been identified, decoded and hashed. It deliberately
|
||||
// does not trust an HTTP Content-Type header.
|
||||
func ValidateReadyAttachmentMetadata(
|
||||
filename string,
|
||||
contentType string,
|
||||
sizeBytes int64,
|
||||
sha256Hex string,
|
||||
width int,
|
||||
height int,
|
||||
) error {
|
||||
if err := ValidateAttachmentFilename(filename); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ValidateAttachmentContentType(contentType); err != nil {
|
||||
return err
|
||||
}
|
||||
if sizeBytes < 1 || sizeBytes > MaxAttachmentSizeBytes {
|
||||
return fmt.Errorf("attachment size must be between 1 and %d bytes", MaxAttachmentSizeBytes)
|
||||
}
|
||||
if width < 1 || height < 1 {
|
||||
return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels)
|
||||
}
|
||||
if int64(width) > int64(MaxAttachmentPixels)/int64(height) {
|
||||
return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels)
|
||||
}
|
||||
if len(sha256Hex) != 64 || sha256Hex != strings.ToLower(sha256Hex) {
|
||||
return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters")
|
||||
}
|
||||
decodedHash, err := hex.DecodeString(sha256Hex)
|
||||
if err != nil || len(decodedHash) != 32 {
|
||||
return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateReadyAttachmentMetadata(t *testing.T) {
|
||||
validHash := strings.Repeat("ab", 32)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
filename string
|
||||
contentType string
|
||||
sizeBytes int64
|
||||
sha256 string
|
||||
width int
|
||||
height int
|
||||
wantError bool
|
||||
}{
|
||||
{name: "png", filename: "кот.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1},
|
||||
{name: "jpeg at limits", filename: strings.Repeat("界", 255), contentType: "image/jpeg", sizeBytes: MaxAttachmentSizeBytes, sha256: validHash, width: 8_000, height: 5_000},
|
||||
{name: "empty filename", filename: "", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "blank filename", filename: " ", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "dot segment", filename: "..", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "forward slash", filename: "folder/image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "backslash", filename: `folder\\image.png`, contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "control character", filename: "image\n.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "C1 control character", filename: "image\u0085.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "invalid UTF-8", filename: string([]byte{'x', 0xff, '.', 'p', 'n', 'g'}), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "filename too long", filename: strings.Repeat("a", 256), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "unsupported MIME", filename: "image.gif", contentType: "image/gif", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "empty file", filename: "image.png", contentType: "image/png", sizeBytes: 0, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "file too large", filename: "image.png", contentType: "image/png", sizeBytes: MaxAttachmentSizeBytes + 1, sha256: validHash, width: 1, height: 1, wantError: true},
|
||||
{name: "zero width", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 0, height: 1, wantError: true},
|
||||
{name: "too many pixels", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 8_001, height: 5_000, wantError: true},
|
||||
{name: "dimension multiplication cannot overflow", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: math.MaxInt, height: math.MaxInt, wantError: true},
|
||||
{name: "short hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: "ab", width: 1, height: 1, wantError: true},
|
||||
{name: "uppercase hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.ToUpper(validHash), width: 1, height: 1, wantError: true},
|
||||
{name: "non-hex hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.Repeat("zz", 32), width: 1, height: 1, wantError: true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := ValidateReadyAttachmentMetadata(
|
||||
testCase.filename,
|
||||
testCase.contentType,
|
||||
testCase.sizeBytes,
|
||||
testCase.sha256,
|
||||
testCase.width,
|
||||
testCase.height,
|
||||
)
|
||||
if (err != nil) != testCase.wantError {
|
||||
t.Fatalf("ValidateReadyAttachmentMetadata() error = %v, wantError %v", err, testCase.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,12 @@ package domain
|
||||
import "time"
|
||||
|
||||
type Message struct {
|
||||
ID string
|
||||
ChatID string
|
||||
SenderID string
|
||||
Text string
|
||||
CreatedAt time.Time
|
||||
ID string
|
||||
ChatID string
|
||||
SenderID string
|
||||
Text string
|
||||
AttachmentIDs []string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type MessageCursor struct {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateMessageInput(t *testing.T) {
|
||||
validAttachment := "11111111-1111-4111-8111-111111111111"
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
text string
|
||||
attachmentIDs []string
|
||||
wantError bool
|
||||
}{
|
||||
{name: "text only", text: "hello"},
|
||||
{name: "attachment only", attachmentIDs: []string{validAttachment}},
|
||||
{name: "whitespace with attachment", text: " ", attachmentIDs: []string{validAttachment}},
|
||||
{name: "empty", wantError: true},
|
||||
{name: "whitespace only", text: " ", wantError: true},
|
||||
{name: "text too long", text: strings.Repeat("界", 4001), wantError: true},
|
||||
{name: "invalid attachment uuid", text: "hello", attachmentIDs: []string{"not-a-uuid"}, wantError: true},
|
||||
{
|
||||
name: "duplicate attachment uuid ignores case",
|
||||
text: "hello",
|
||||
attachmentIDs: []string{
|
||||
"AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA",
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := ValidateMessageInput(testCase.text, testCase.attachmentIDs)
|
||||
if (err != nil) != testCase.wantError {
|
||||
t.Fatalf("ValidateMessageInput() error = %v, wantError %v", err, testCase.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMessageInputAttachmentLimit(t *testing.T) {
|
||||
attachmentIDs := []string{
|
||||
"00000000-0000-4000-8000-000000000001",
|
||||
"00000000-0000-4000-8000-000000000002",
|
||||
"00000000-0000-4000-8000-000000000003",
|
||||
"00000000-0000-4000-8000-000000000004",
|
||||
"00000000-0000-4000-8000-000000000005",
|
||||
"00000000-0000-4000-8000-000000000006",
|
||||
"00000000-0000-4000-8000-000000000007",
|
||||
"00000000-0000-4000-8000-000000000008",
|
||||
"00000000-0000-4000-8000-000000000009",
|
||||
"00000000-0000-4000-8000-000000000010",
|
||||
}
|
||||
|
||||
if err := ValidateMessageInput("", attachmentIDs); err != nil {
|
||||
t.Fatalf("10 attachments must be accepted: %v", err)
|
||||
}
|
||||
|
||||
attachmentIDs = append(
|
||||
attachmentIDs,
|
||||
"00000000-0000-4000-8000-000000000011",
|
||||
)
|
||||
if err := ValidateMessageInput("hello", attachmentIDs); err == nil {
|
||||
t.Fatal("11 attachments must be rejected")
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const MaxMessageAttachments = 10
|
||||
|
||||
var usernameRegex = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
|
||||
func ValidateUser(username, displayName string) error {
|
||||
@@ -65,13 +67,35 @@ func ValidateChatInput(title string, memberIDs []string, callerID string) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateMessageText(text string) error {
|
||||
func ValidateMessageInput(text string, attachmentIDs []string) error {
|
||||
textRunes := utf8.RuneCountInString(text)
|
||||
if textRunes < 1 || textRunes > 4000 {
|
||||
return fmt.Errorf("message text length must be between 1 and 4000 code points")
|
||||
if textRunes > 4000 {
|
||||
return fmt.Errorf("message text length must be at most 4000 code points")
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return fmt.Errorf("message text cannot be only whitespace")
|
||||
if len(attachmentIDs) > MaxMessageAttachments {
|
||||
return fmt.Errorf("attachment_ids must contain at most %d items", MaxMessageAttachments)
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(attachmentIDs))
|
||||
for _, attachmentID := range attachmentIDs {
|
||||
parsedID, err := uuid.Parse(attachmentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid attachment uuid: %s", attachmentID)
|
||||
}
|
||||
canonicalID := parsedID.String()
|
||||
if _, exists := seen[canonicalID]; exists {
|
||||
return fmt.Errorf("duplicate attachment_id: %s", attachmentID)
|
||||
}
|
||||
seen[canonicalID] = struct{}{}
|
||||
}
|
||||
|
||||
if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 {
|
||||
return fmt.Errorf("message must contain non-blank text or at least one attachment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateMessageText remains for callers that create text-only messages.
|
||||
func ValidateMessageText(text string) error {
|
||||
return ValidateMessageInput(text, nil)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/adapter/objectstore"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
IssueOrphanObject = "orphan_object"
|
||||
IssueMissingObject = "missing_object"
|
||||
IssueStalePending = "stale_pending"
|
||||
IssueFailedObject = "failed_object"
|
||||
IssueReferencedMissing = "referenced_missing_object"
|
||||
)
|
||||
|
||||
type AttachmentRecord struct {
|
||||
ID string
|
||||
ObjectKey string
|
||||
Status domain.AttachmentStatus
|
||||
UpdatedAt time.Time
|
||||
Referenced bool
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
ListAttachmentInventory(ctx context.Context) ([]AttachmentRecord, error)
|
||||
MarkAttachmentFailedIfUnreferenced(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
expectedStatus domain.AttachmentStatus,
|
||||
updatedBefore time.Time,
|
||||
errorCode string,
|
||||
updatedAt time.Time,
|
||||
) (bool, error)
|
||||
}
|
||||
|
||||
type ObjectInventory interface {
|
||||
List(ctx context.Context, prefix string) ([]objectstore.StoredObject, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
type Issue struct {
|
||||
Kind string `json:"kind"`
|
||||
AttachmentID string `json:"attachment_id,omitempty"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
Protected bool `json:"protected,omitempty"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
GracePeriod time.Duration `json:"-"`
|
||||
GracePeriodSeconds int64 `json:"grace_period_seconds"`
|
||||
ScannedAttachments int `json:"scanned_attachments"`
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
Issues []Issue `json:"issues"`
|
||||
DeletedObjects int `json:"deleted_objects"`
|
||||
FailedAttachments int `json:"failed_attachments"`
|
||||
Duration time.Duration `json:"-"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
metadata MetadataStore
|
||||
objects ObjectInventory
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(metadata MetadataStore, objects ObjectInventory) (*Reconciler, error) {
|
||||
if metadata == nil || objects == nil {
|
||||
return nil, fmt.Errorf("metadata and object stores are required")
|
||||
}
|
||||
return &Reconciler{
|
||||
metadata: metadata,
|
||||
objects: objects,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) Run(
|
||||
ctx context.Context,
|
||||
gracePeriod time.Duration,
|
||||
apply bool,
|
||||
) (Result, error) {
|
||||
if gracePeriod <= 0 {
|
||||
return Result{}, fmt.Errorf("grace period must be positive")
|
||||
}
|
||||
startedAt := r.now().UTC()
|
||||
cutoff := startedAt.Add(-gracePeriod)
|
||||
result := Result{
|
||||
DryRun: !apply,
|
||||
GracePeriod: gracePeriod,
|
||||
GracePeriodSeconds: int64(gracePeriod / time.Second),
|
||||
Issues: make([]Issue, 0),
|
||||
}
|
||||
|
||||
attachments, err := r.metadata.ListAttachmentInventory(ctx)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list attachment metadata: %w", err)
|
||||
}
|
||||
objects, err := r.objects.List(ctx, "attachments/")
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list attachment objects: %w", err)
|
||||
}
|
||||
result.ScannedAttachments = len(attachments)
|
||||
result.ScannedObjects = len(objects)
|
||||
|
||||
attachmentsByKey := make(map[string]AttachmentRecord, len(attachments))
|
||||
for _, attachment := range attachments {
|
||||
attachmentsByKey[attachment.ObjectKey] = attachment
|
||||
}
|
||||
objectsByKey := make(map[string]objectstore.StoredObject, len(objects))
|
||||
for _, object := range objects {
|
||||
objectsByKey[object.Key] = object
|
||||
if _, exists := attachmentsByKey[object.Key]; exists {
|
||||
continue
|
||||
}
|
||||
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
issue := Issue{Kind: IssueOrphanObject, ObjectKey: object.Key, Action: "would_delete"}
|
||||
if apply {
|
||||
if err = r.objects.Delete(ctx, object.Key); err != nil {
|
||||
return result, fmt.Errorf("delete orphan object %q: %w", object.Key, err)
|
||||
}
|
||||
issue.Action = "deleted"
|
||||
result.DeletedObjects++
|
||||
}
|
||||
result.Issues = append(result.Issues, issue)
|
||||
}
|
||||
|
||||
for _, attachment := range attachments {
|
||||
if attachment.UpdatedAt.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
_, objectExists := objectsByKey[attachment.ObjectKey]
|
||||
switch {
|
||||
case attachment.Status == domain.AttachmentStatusReady && !objectExists:
|
||||
issue := Issue{
|
||||
Kind: IssueMissingObject,
|
||||
AttachmentID: attachment.ID,
|
||||
ObjectKey: attachment.ObjectKey,
|
||||
Protected: attachment.Referenced,
|
||||
Action: "would_mark_failed",
|
||||
}
|
||||
if attachment.Referenced {
|
||||
issue.Kind = IssueReferencedMissing
|
||||
issue.Action = "manual_repair_required"
|
||||
} else if apply {
|
||||
changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced(
|
||||
ctx, attachment.ID, attachment.Status, cutoff,
|
||||
"reconciler_missing_object", startedAt,
|
||||
)
|
||||
if markError != nil {
|
||||
return result, fmt.Errorf("mark missing attachment %q failed: %w", attachment.ID, markError)
|
||||
}
|
||||
if changed {
|
||||
issue.Action = "marked_failed"
|
||||
result.FailedAttachments++
|
||||
} else {
|
||||
issue.Action = "skipped_concurrent_change"
|
||||
}
|
||||
}
|
||||
result.Issues = append(result.Issues, issue)
|
||||
|
||||
case attachment.Status == domain.AttachmentStatusPending:
|
||||
issue := Issue{
|
||||
Kind: IssueStalePending, AttachmentID: attachment.ID,
|
||||
ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced,
|
||||
Action: "would_mark_failed",
|
||||
}
|
||||
if attachment.Referenced {
|
||||
issue.Action = "manual_repair_required"
|
||||
} else if apply {
|
||||
changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced(
|
||||
ctx, attachment.ID, attachment.Status, cutoff,
|
||||
"reconciler_stale_pending", startedAt,
|
||||
)
|
||||
if markError != nil {
|
||||
return result, fmt.Errorf("mark stale attachment %q failed: %w", attachment.ID, markError)
|
||||
}
|
||||
if changed {
|
||||
issue.Action = "marked_failed"
|
||||
result.FailedAttachments++
|
||||
if objectExists {
|
||||
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
|
||||
return result, fmt.Errorf("delete stale pending object %q: %w", attachment.ObjectKey, err)
|
||||
}
|
||||
result.DeletedObjects++
|
||||
}
|
||||
} else {
|
||||
issue.Action = "skipped_concurrent_change"
|
||||
}
|
||||
}
|
||||
result.Issues = append(result.Issues, issue)
|
||||
|
||||
case attachment.Status == domain.AttachmentStatusFailed && objectExists:
|
||||
issue := Issue{
|
||||
Kind: IssueFailedObject, AttachmentID: attachment.ID,
|
||||
ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced,
|
||||
Action: "would_delete",
|
||||
}
|
||||
if attachment.Referenced {
|
||||
issue.Action = "manual_repair_required"
|
||||
} else if apply {
|
||||
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
|
||||
return result, fmt.Errorf("delete failed attachment object %q: %w", attachment.ObjectKey, err)
|
||||
}
|
||||
issue.Action = "deleted"
|
||||
result.DeletedObjects++
|
||||
}
|
||||
result.Issues = append(result.Issues, issue)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(result.Issues, func(i, j int) bool {
|
||||
if result.Issues[i].ObjectKey == result.Issues[j].ObjectKey {
|
||||
return result.Issues[i].Kind < result.Issues[j].Kind
|
||||
}
|
||||
return result.Issues[i].ObjectKey < result.Issues[j].ObjectKey
|
||||
})
|
||||
result.Duration = r.now().UTC().Sub(startedAt)
|
||||
result.DurationMillis = result.Duration.Milliseconds()
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/adapter/objectstore"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
type fakeMetadataStore struct {
|
||||
records []AttachmentRecord
|
||||
markError error
|
||||
skipMark bool
|
||||
}
|
||||
|
||||
func (store *fakeMetadataStore) ListAttachmentInventory(
|
||||
context.Context,
|
||||
) ([]AttachmentRecord, error) {
|
||||
return append([]AttachmentRecord(nil), store.records...), nil
|
||||
}
|
||||
|
||||
func (store *fakeMetadataStore) MarkAttachmentFailedIfUnreferenced(
|
||||
_ context.Context,
|
||||
attachmentID string,
|
||||
expectedStatus domain.AttachmentStatus,
|
||||
updatedBefore time.Time,
|
||||
errorCode string,
|
||||
updatedAt time.Time,
|
||||
) (bool, error) {
|
||||
if store.markError != nil {
|
||||
return false, store.markError
|
||||
}
|
||||
if store.skipMark {
|
||||
return false, nil
|
||||
}
|
||||
for index := range store.records {
|
||||
record := &store.records[index]
|
||||
if record.ID != attachmentID || record.Status != expectedStatus ||
|
||||
record.UpdatedAt.After(updatedBefore) || record.Referenced {
|
||||
continue
|
||||
}
|
||||
record.Status = domain.AttachmentStatusFailed
|
||||
record.UpdatedAt = updatedAt
|
||||
_ = errorCode
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func TestReconcilerDoesNotDeleteAfterConcurrentMetadataChange(t *testing.T) {
|
||||
old := time.Now().UTC().Add(-2 * time.Hour)
|
||||
metadata := &fakeMetadataStore{
|
||||
records: []AttachmentRecord{{
|
||||
ID: "pending-race", ObjectKey: "attachments/pending-race",
|
||||
Status: domain.AttachmentStatusPending, UpdatedAt: old,
|
||||
}},
|
||||
skipMark: true,
|
||||
}
|
||||
objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{
|
||||
"attachments/pending-race": {Key: "attachments/pending-race", LastModified: old},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := reconciler.Run(context.Background(), time.Hour, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.DeletedObjects != 0 || result.FailedAttachments != 0 {
|
||||
t.Fatalf("concurrent metadata change must suppress deletion: %+v", result)
|
||||
}
|
||||
if _, exists := objects.objects["attachments/pending-race"]; !exists {
|
||||
t.Fatal("reconciler deleted an object after losing the metadata race")
|
||||
}
|
||||
if len(result.Issues) != 1 || result.Issues[0].Action != "skipped_concurrent_change" {
|
||||
t.Fatalf("unexpected race result: %+v", result.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeObjectInventory struct {
|
||||
objects map[string]objectstore.StoredObject
|
||||
deleteError error
|
||||
}
|
||||
|
||||
func (store *fakeObjectInventory) List(
|
||||
context.Context,
|
||||
string,
|
||||
) ([]objectstore.StoredObject, error) {
|
||||
objects := make([]objectstore.StoredObject, 0, len(store.objects))
|
||||
for _, object := range store.objects {
|
||||
objects = append(objects, object)
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (store *fakeObjectInventory) Delete(_ context.Context, key string) error {
|
||||
if store.deleteError != nil {
|
||||
return store.deleteError
|
||||
}
|
||||
delete(store.objects, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestReconcilerDryRunApplyAndRepeat(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
old := now.Add(-2 * time.Hour)
|
||||
fresh := now.Add(-5 * time.Minute)
|
||||
metadata := &fakeMetadataStore{records: []AttachmentRecord{
|
||||
{ID: "ready-present", ObjectKey: "attachments/ready-present", Status: domain.AttachmentStatusReady, UpdatedAt: old},
|
||||
{ID: "ready-missing", ObjectKey: "attachments/ready-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old},
|
||||
{ID: "ready-used-missing", ObjectKey: "attachments/ready-used-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old, Referenced: true},
|
||||
{ID: "pending-old", ObjectKey: "attachments/pending-old", Status: domain.AttachmentStatusPending, UpdatedAt: old},
|
||||
{ID: "failed-old", ObjectKey: "attachments/failed-old", Status: domain.AttachmentStatusFailed, UpdatedAt: old},
|
||||
{ID: "pending-fresh", ObjectKey: "attachments/pending-fresh", Status: domain.AttachmentStatusPending, UpdatedAt: fresh},
|
||||
}}
|
||||
objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{
|
||||
"attachments/ready-present": {Key: "attachments/ready-present", LastModified: old},
|
||||
"attachments/pending-old": {Key: "attachments/pending-old", LastModified: old},
|
||||
"attachments/failed-old": {Key: "attachments/failed-old", LastModified: old},
|
||||
"attachments/orphan-old": {Key: "attachments/orphan-old", LastModified: old},
|
||||
"attachments/orphan-fresh": {Key: "attachments/orphan-fresh", LastModified: fresh},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dryRun, err := reconciler.Run(context.Background(), time.Hour, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !dryRun.DryRun || len(dryRun.Issues) != 5 ||
|
||||
dryRun.DeletedObjects != 0 || dryRun.FailedAttachments != 0 {
|
||||
t.Fatalf("unexpected dry-run result: %+v", dryRun)
|
||||
}
|
||||
if len(objects.objects) != 5 {
|
||||
t.Fatal("dry-run changed object storage")
|
||||
}
|
||||
for _, record := range metadata.records {
|
||||
if record.ID == "ready-missing" && record.Status != domain.AttachmentStatusReady {
|
||||
t.Fatal("dry-run changed metadata")
|
||||
}
|
||||
}
|
||||
|
||||
applied, err := reconciler.Run(context.Background(), time.Hour, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if applied.DeletedObjects != 3 || applied.FailedAttachments != 2 {
|
||||
t.Fatalf("unexpected apply result: %+v", applied)
|
||||
}
|
||||
if _, exists := objects.objects["attachments/orphan-old"]; exists {
|
||||
t.Fatal("old orphan was not deleted")
|
||||
}
|
||||
if _, exists := objects.objects["attachments/orphan-fresh"]; !exists {
|
||||
t.Fatal("fresh orphan must be protected by grace period")
|
||||
}
|
||||
if _, exists := objects.objects["attachments/ready-present"]; !exists {
|
||||
t.Fatal("healthy object was deleted")
|
||||
}
|
||||
for _, issue := range applied.Issues {
|
||||
if issue.AttachmentID == "ready-used-missing" &&
|
||||
(!issue.Protected || issue.Action != "manual_repair_required") {
|
||||
t.Fatalf("referenced attachment was not protected: %+v", issue)
|
||||
}
|
||||
}
|
||||
|
||||
repeated, err := reconciler.Run(context.Background(), time.Hour, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repeated.DeletedObjects != 0 || repeated.FailedAttachments != 0 {
|
||||
t.Fatalf("repeat must be mutation-idempotent: %+v", repeated)
|
||||
}
|
||||
if len(repeated.Issues) != 1 || !repeated.Issues[0].Protected {
|
||||
t.Fatalf("repeat should retain only protected manual issue: %+v", repeated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerReportsMutationFailures(t *testing.T) {
|
||||
old := time.Now().UTC().Add(-2 * time.Hour)
|
||||
metadata := &fakeMetadataStore{records: []AttachmentRecord{
|
||||
{ID: "missing", ObjectKey: "attachments/missing", Status: domain.AttachmentStatusReady, UpdatedAt: old},
|
||||
}}
|
||||
metadata.markError = errors.New("injected database failure")
|
||||
objects := &fakeObjectInventory{objects: make(map[string]objectstore.StoredObject)}
|
||||
reconciler, err := New(metadata, objects)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil {
|
||||
t.Fatal("injected database mutation failure was hidden")
|
||||
}
|
||||
|
||||
metadata.records = nil
|
||||
metadata.markError = nil
|
||||
objects.objects["attachments/orphan"] = objectstore.StoredObject{
|
||||
Key: "attachments/orphan", LastModified: old,
|
||||
}
|
||||
objects.deleteError = errors.New("injected object-store failure")
|
||||
if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil {
|
||||
t.Fatal("injected object-store mutation failure was hidden")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
type AttachmentVariantResponse struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type AttachmentResponse struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status domain.AttachmentStatus `json:"status"`
|
||||
Variants []AttachmentVariantResponse `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func AttachmentFromDomain(attachment domain.Attachment) AttachmentResponse {
|
||||
variants := make([]AttachmentVariantResponse, 0, len(attachment.Variants))
|
||||
for _, variant := range attachment.Variants {
|
||||
variants = append(variants, AttachmentVariantResponse{
|
||||
Name: variant.Name,
|
||||
ContentType: variant.ContentType,
|
||||
SizeBytes: variant.SizeBytes,
|
||||
Width: variant.Width,
|
||||
Height: variant.Height,
|
||||
SHA256: variant.SHA256,
|
||||
})
|
||||
}
|
||||
return AttachmentResponse{
|
||||
ID: attachment.ID,
|
||||
ChatID: attachment.ChatID,
|
||||
OwnerID: attachment.OwnerID,
|
||||
Filename: attachment.Filename,
|
||||
ContentType: attachment.ContentType,
|
||||
SizeBytes: attachment.SizeBytes,
|
||||
SHA256: attachment.SHA256,
|
||||
Status: attachment.Status,
|
||||
Variants: variants,
|
||||
ErrorCode: attachment.ErrorCode,
|
||||
CreatedAt: attachment.CreatedAt,
|
||||
UpdatedAt: attachment.UpdatedAt,
|
||||
}
|
||||
}
|
||||
@@ -7,15 +7,17 @@ import (
|
||||
)
|
||||
|
||||
type CreateMessageRequest struct {
|
||||
Text string `json:"text"`
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type MessageResponse struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type MessagePageResponse struct {
|
||||
@@ -24,9 +26,14 @@ type MessagePageResponse struct {
|
||||
}
|
||||
|
||||
func MessageFromDomain(message domain.Message) MessageResponse {
|
||||
attachmentIDs := append([]string(nil), message.AttachmentIDs...)
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = make([]string, 0)
|
||||
}
|
||||
|
||||
return MessageResponse{
|
||||
ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID,
|
||||
Text: message.Text, CreatedAt: message.CreatedAt,
|
||||
Text: message.Text, AttachmentIDs: attachmentIDs, CreatedAt: message.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
ActorID: currentUser.ID,
|
||||
ChatID: chatID,
|
||||
Text: request.Text,
|
||||
AttachmentIDs: request.AttachmentIDs,
|
||||
Method: http.MethodPost,
|
||||
CanonicalPath: "/api/v1/chats/" + chatID + "/messages",
|
||||
IdempotencyKey: idempotencyKey,
|
||||
@@ -53,6 +54,11 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
"Idempotency-Key is already used for another request", requestID)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, application.ErrAttachmentNotReady) {
|
||||
response.WriteError(w, http.StatusConflict, response.CodeConflict,
|
||||
"attachment is not ready", requestID)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, application.ErrInvalidInput) {
|
||||
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
|
||||
strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": "), requestID)
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"mtusi2026lab4/internal/application"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
"mtusi2026lab4/internal/transport/http/dto"
|
||||
"mtusi2026lab4/internal/transport/http/middleware"
|
||||
"mtusi2026lab4/internal/transport/http/response"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
currentUser := middleware.GetCurrentUser(r.Context())
|
||||
attachmentID, ok := parseAttachmentID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
attachment, err := h.service.GetAttachment(r.Context(), attachmentID, currentUser.ID)
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
response.WriteError(w, http.StatusNotFound, response.CodeNotFound,
|
||||
"attachment not found", requestID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable,
|
||||
"service is temporarily unavailable", requestID)
|
||||
return
|
||||
}
|
||||
response.WriteJSON(w, http.StatusOK, dto.AttachmentFromDomain(attachment), requestID)
|
||||
}
|
||||
|
||||
func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
currentUser := middleware.GetCurrentUser(r.Context())
|
||||
attachmentID, ok := parseAttachmentID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
download, err := h.service.DownloadAttachment(
|
||||
r.Context(), attachmentID, currentUser.ID, r.URL.Query().Get("variant"),
|
||||
)
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
response.WriteError(w, http.StatusNotFound, response.CodeNotFound,
|
||||
"attachment or variant not found", requestID)
|
||||
return
|
||||
case errors.Is(err, application.ErrAttachmentNotReady):
|
||||
response.WriteError(w, http.StatusConflict, response.CodeConflict,
|
||||
"attachment is not ready", requestID)
|
||||
return
|
||||
case errors.Is(err, application.ErrStorageUnavailable):
|
||||
response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable,
|
||||
"object storage is temporarily unavailable", requestID)
|
||||
return
|
||||
case err != nil:
|
||||
response.WriteError(w, http.StatusInternalServerError, response.CodeInternalError,
|
||||
"unexpected internal error", requestID)
|
||||
return
|
||||
}
|
||||
defer download.Body.Close()
|
||||
|
||||
w.Header().Set("Content-Type", download.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(download.SizeBytes, 10))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType(
|
||||
"attachment", map[string]string{"filename": download.Filename},
|
||||
))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.Copy(w, download.Body)
|
||||
}
|
||||
|
||||
func parseAttachmentID(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
requestID string,
|
||||
) (string, bool) {
|
||||
parsedID, err := uuid.Parse(r.PathValue("attachment_id"))
|
||||
if err != nil {
|
||||
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
|
||||
"invalid attachment_id uuid", requestID)
|
||||
return "", false
|
||||
}
|
||||
return parsedID.String(), true
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"mtusi2026lab4/internal/application"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
"mtusi2026lab4/internal/transport/http/dto"
|
||||
"mtusi2026lab4/internal/transport/http/middleware"
|
||||
"mtusi2026lab4/internal/transport/http/response"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
currentUser := middleware.GetCurrentUser(r.Context())
|
||||
|
||||
parsedChatID, err := uuid.Parse(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
|
||||
"invalid chat_id uuid", requestID)
|
||||
return
|
||||
}
|
||||
filename := r.URL.Query().Get("filename")
|
||||
if filename == "" {
|
||||
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
|
||||
"filename query parameter is required", requestID)
|
||||
return
|
||||
}
|
||||
|
||||
contentType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || len(parameters) != 0 ||
|
||||
(contentType != "image/png" && contentType != "image/jpeg") {
|
||||
response.WriteError(w, http.StatusUnsupportedMediaType,
|
||||
response.CodeUnsupportedMediaType,
|
||||
"Content-Type must be image/png or image/jpeg", requestID)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > domain.MaxAttachmentSizeBytes {
|
||||
response.WriteError(w, http.StatusRequestEntityTooLarge,
|
||||
response.CodePayloadTooLarge,
|
||||
"attachment exceeds 10 MiB", requestID)
|
||||
return
|
||||
}
|
||||
if r.Body == nil {
|
||||
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
|
||||
"attachment body is required", requestID)
|
||||
return
|
||||
}
|
||||
defer r.Body.Close()
|
||||
|
||||
attachment, err := h.service.UploadAttachment(
|
||||
r.Context(),
|
||||
application.UploadAttachmentCommand{
|
||||
ActorID: currentUser.ID,
|
||||
ChatID: parsedChatID.String(),
|
||||
Filename: filename,
|
||||
DeclaredContentType: contentType,
|
||||
Body: r.Body,
|
||||
},
|
||||
)
|
||||
switch {
|
||||
case errors.Is(err, application.ErrPayloadTooLarge):
|
||||
response.WriteError(w, http.StatusRequestEntityTooLarge,
|
||||
response.CodePayloadTooLarge,
|
||||
"attachment exceeds 10 MiB or 40 million pixels", requestID)
|
||||
return
|
||||
case errors.Is(err, application.ErrUnsupportedMediaType), errors.Is(err, application.ErrInvalidImage):
|
||||
response.WriteError(w, http.StatusUnsupportedMediaType,
|
||||
response.CodeUnsupportedMediaType,
|
||||
"body is not a valid PNG/JPEG matching the declared Content-Type", requestID)
|
||||
return
|
||||
case errors.Is(err, application.ErrInvalidInput):
|
||||
message := strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": ")
|
||||
response.WriteError(w, http.StatusBadRequest,
|
||||
response.CodeInvalidRequest, message, requestID)
|
||||
return
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
response.WriteError(w, http.StatusNotFound,
|
||||
response.CodeNotFound, "chat not found", requestID)
|
||||
return
|
||||
case errors.Is(err, application.ErrStorageUnavailable):
|
||||
response.WriteError(w, http.StatusServiceUnavailable,
|
||||
response.CodeUnavailable, "object storage is temporarily unavailable", requestID)
|
||||
return
|
||||
case err != nil:
|
||||
response.WriteError(w, http.StatusServiceUnavailable,
|
||||
response.CodeUnavailable, "service is temporarily unavailable", requestID)
|
||||
return
|
||||
}
|
||||
|
||||
response.WriteJSON(w, http.StatusCreated, dto.AttachmentFromDomain(attachment), requestID)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
BEGIN;
|
||||
|
||||
-- Downgrading would destroy the only content of attachment-only messages.
|
||||
-- Refuse instead of silently losing valid lab4 data.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM messages
|
||||
WHERE btrim(text) = ''
|
||||
) THEN
|
||||
RAISE EXCEPTION
|
||||
'cannot downgrade while attachment-only messages exist';
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER message_attachments_payload_check ON message_attachments;
|
||||
DROP TRIGGER messages_payload_check ON messages;
|
||||
DROP FUNCTION enforce_message_payload_from_link();
|
||||
DROP FUNCTION enforce_message_payload_from_message();
|
||||
|
||||
DROP TABLE message_attachments;
|
||||
DROP TABLE attachments;
|
||||
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_id_chat_unique,
|
||||
DROP CONSTRAINT messages_text_length;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length
|
||||
CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
ADD CONSTRAINT messages_text_not_blank
|
||||
CHECK (btrim(text) <> '');
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,191 @@
|
||||
BEGIN;
|
||||
|
||||
-- Text may be empty only when the same transaction links at least one
|
||||
-- attachment. A deferred constraint trigger below enforces that invariant.
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_length,
|
||||
DROP CONSTRAINT messages_text_not_blank;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length
|
||||
CHECK (char_length(text) BETWEEN 0 AND 4000),
|
||||
ADD CONSTRAINT messages_id_chat_unique
|
||||
UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL,
|
||||
owner_id UUID NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 BYTEA NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT attachments_id_chat_unique
|
||||
UNIQUE (id, chat_id),
|
||||
|
||||
CONSTRAINT attachments_object_key_unique
|
||||
UNIQUE (object_key),
|
||||
|
||||
CONSTRAINT attachments_owner_membership_foreign_key
|
||||
FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id)
|
||||
ON DELETE RESTRICT,
|
||||
|
||||
CONSTRAINT attachments_object_key_not_blank
|
||||
CHECK (btrim(object_key) <> ''),
|
||||
|
||||
CONSTRAINT attachments_filename_length
|
||||
CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
|
||||
CONSTRAINT attachments_filename_safe
|
||||
CHECK (
|
||||
btrim(filename) <> ''
|
||||
AND filename NOT IN ('.', '..')
|
||||
AND position('/' in filename) = 0
|
||||
AND position(chr(92) in filename) = 0
|
||||
AND filename !~ '[[:cntrl:]]'
|
||||
),
|
||||
|
||||
CONSTRAINT attachments_content_type_supported
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
|
||||
CONSTRAINT attachments_size_range
|
||||
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
|
||||
CONSTRAINT attachments_sha256_length
|
||||
CHECK (octet_length(sha256) = 32),
|
||||
|
||||
CONSTRAINT attachments_dimensions_positive
|
||||
CHECK (width > 0 AND height > 0),
|
||||
|
||||
CONSTRAINT attachments_pixel_limit
|
||||
CHECK (width::BIGINT * height::BIGINT <= 40000000),
|
||||
|
||||
CONSTRAINT attachments_status_supported
|
||||
CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
|
||||
CONSTRAINT attachments_status_error_consistency
|
||||
CHECK (
|
||||
(status = 'failed' AND error_code IS NOT NULL AND btrim(error_code) <> '')
|
||||
OR
|
||||
(status IN ('pending', 'ready') AND error_code IS NULL)
|
||||
),
|
||||
|
||||
CONSTRAINT attachments_updated_after_creation
|
||||
CHECK (updated_at >= created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_created_at_id_idx
|
||||
ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX attachments_status_updated_at_idx
|
||||
ON attachments (status, updated_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL,
|
||||
position SMALLINT NOT NULL,
|
||||
|
||||
CONSTRAINT message_attachments_primary_key
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
|
||||
CONSTRAINT message_attachments_position_unique
|
||||
UNIQUE (message_id, position),
|
||||
|
||||
CONSTRAINT message_attachments_position_range
|
||||
CHECK (position BETWEEN 0 AND 9),
|
||||
|
||||
CONSTRAINT message_attachments_message_foreign_key
|
||||
FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages (id, chat_id)
|
||||
ON DELETE CASCADE,
|
||||
|
||||
CONSTRAINT message_attachments_attachment_foreign_key
|
||||
FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments (id, chat_id)
|
||||
ON DELETE RESTRICT
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_id_idx
|
||||
ON message_attachments (attachment_id, message_id);
|
||||
|
||||
CREATE FUNCTION enforce_message_payload_from_message()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM messages AS message
|
||||
WHERE message.id = NEW.id
|
||||
AND btrim(message.text) = ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM message_attachments AS link
|
||||
WHERE link.message_id = message.id
|
||||
)
|
||||
) THEN
|
||||
RAISE EXCEPTION USING
|
||||
ERRCODE = '23514',
|
||||
CONSTRAINT = 'messages_content_required',
|
||||
MESSAGE = 'message must contain non-blank text or an attachment';
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION enforce_message_payload_from_link()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
affected_message_id UUID;
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' THEN
|
||||
affected_message_id := OLD.message_id;
|
||||
ELSE
|
||||
affected_message_id := NEW.message_id;
|
||||
END IF;
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM messages AS message
|
||||
WHERE message.id = affected_message_id
|
||||
AND btrim(message.text) = ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM message_attachments AS link
|
||||
WHERE link.message_id = message.id
|
||||
)
|
||||
) THEN
|
||||
RAISE EXCEPTION USING
|
||||
ERRCODE = '23514',
|
||||
CONSTRAINT = 'messages_content_required',
|
||||
MESSAGE = 'message must contain non-blank text or an attachment';
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE CONSTRAINT TRIGGER messages_payload_check
|
||||
AFTER INSERT OR UPDATE ON messages
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION enforce_message_payload_from_message();
|
||||
|
||||
CREATE CONSTRAINT TRIGGER message_attachments_payload_check
|
||||
AFTER INSERT OR UPDATE OR DELETE ON message_attachments
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION enforce_message_payload_from_link();
|
||||
|
||||
COMMIT;
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
secrets_dir="$project_root/secrets"
|
||||
|
||||
command -v openssl >/dev/null 2>&1 || {
|
||||
printf '%s\n' "openssl is required to generate local secrets" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
umask 077
|
||||
mkdir -p "$secrets_dir"
|
||||
chmod 700 "$secrets_dir"
|
||||
|
||||
write_if_missing() {
|
||||
destination=$1
|
||||
value=$2
|
||||
if [ ! -e "$destination" ]; then
|
||||
printf '%s' "$value" > "$destination"
|
||||
fi
|
||||
chmod 600 "$destination"
|
||||
}
|
||||
|
||||
random_hex() {
|
||||
openssl rand -hex "$1"
|
||||
}
|
||||
|
||||
write_if_missing "$secrets_dir/postgres_password.txt" "$(random_hex 24)"
|
||||
postgres_password=$(cat "$secrets_dir/postgres_password.txt")
|
||||
write_if_missing "$secrets_dir/database_url.txt" \
|
||||
"postgres://messenger:${postgres_password}@postgres:5432/messenger?sslmode=disable"
|
||||
write_if_missing "$secrets_dir/cursor_signing_key.txt" "$(random_hex 32)"
|
||||
write_if_missing "$secrets_dir/minio_root_user.txt" "lab4-root"
|
||||
write_if_missing "$secrets_dir/minio_root_password.txt" "$(random_hex 24)"
|
||||
write_if_missing "$secrets_dir/s3_access_key_id.txt" "lab4-api"
|
||||
write_if_missing "$secrets_dir/s3_secret_access_key.txt" "$(random_hex 24)"
|
||||
|
||||
printf '%s\n' "Local secret files are present with mode 600; existing values were not overwritten."
|
||||
+12
-5
@@ -6,13 +6,20 @@ cd "$project_root"
|
||||
|
||||
env_file=${ENV_FILE:-.env}
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
if [ -f "$env_file" ]; then
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# PostgreSQL хранит все обязательные данные и сессии в named volume.
|
||||
# Redis содержит только восстанавливаемый кеш и намеренно не имеет volume.
|
||||
compose restart postgres redis
|
||||
compose up -d --wait postgres redis
|
||||
# PostgreSQL хранит метаданные и сессии, MinIO — исходные байты вложений;
|
||||
# оба используют named volumes. Redis содержит только восстанавливаемый кеш.
|
||||
compose restart postgres redis minio
|
||||
compose up -d --wait postgres redis minio
|
||||
|
||||
# Повторный bootstrap приватного bucket/policy/user обязан быть идемпотентным.
|
||||
compose run --rm minio-bootstrap >/dev/null
|
||||
|
||||
# up восстанавливает желаемое число реплик, включая ранее вручную остановленную.
|
||||
compose up -d --no-deps api
|
||||
|
||||
+9
-1
@@ -9,7 +9,15 @@ cd "$project_root"
|
||||
env_file=${ENV_FILE:-.env}
|
||||
matched_container=
|
||||
|
||||
for container_id in $(docker compose --env-file "$env_file" ps -a -q api); do
|
||||
compose() {
|
||||
if [ -f "$env_file" ]; then
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
for container_id in $(compose ps -a -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
if [ -n "$matched_container" ]; then
|
||||
|
||||
@@ -8,7 +8,11 @@ import (
|
||||
|
||||
const testCursorSigningKey = "student-tests-cursor-signing-key-at-least-32-bytes"
|
||||
|
||||
func newTestService(repository application.Repository, sessionTTL time.Duration) *application.Service {
|
||||
func newTestService(
|
||||
repository application.Repository,
|
||||
sessionTTL time.Duration,
|
||||
options ...application.ServiceOption,
|
||||
) *application.Service {
|
||||
if sessionTTL <= 0 {
|
||||
sessionTTL = time.Hour
|
||||
}
|
||||
@@ -16,7 +20,7 @@ func newTestService(repository application.Repository, sessionTTL time.Duration)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
service, err := application.NewService(repository, codec, sessionTTL)
|
||||
service, err := application.NewService(repository, codec, sessionTTL, options...)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
package student_tests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
store "mtusi2026lab4/internal/adapter/memory"
|
||||
"mtusi2026lab4/internal/application"
|
||||
"mtusi2026lab4/internal/domain"
|
||||
"mtusi2026lab4/internal/transport/http/dto"
|
||||
"mtusi2026lab4/internal/transport/http/handlers"
|
||||
"mtusi2026lab4/internal/transport/http/middleware"
|
||||
)
|
||||
|
||||
type httpFakeObjectStore struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
}
|
||||
|
||||
func (store *httpFakeObjectStore) Put(_ context.Context, object application.ObjectPut) error {
|
||||
payload, err := io.ReadAll(object.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store.mu.Lock()
|
||||
store.objects[object.Key] = payload
|
||||
store.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *httpFakeObjectStore) Get(_ context.Context, key string) (application.ObjectContent, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
payload, exists := store.objects[key]
|
||||
if !exists {
|
||||
return application.ObjectContent{}, domain.ErrNotFound
|
||||
}
|
||||
copyOfPayload := append([]byte(nil), payload...)
|
||||
return application.ObjectContent{
|
||||
Body: io.NopCloser(bytes.NewReader(copyOfPayload)),
|
||||
SizeBytes: int64(len(copyOfPayload)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store *httpFakeObjectStore) Delete(_ context.Context, key string) error {
|
||||
store.mu.Lock()
|
||||
delete(store.objects, key)
|
||||
store.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupAttachmentUploadServer(
|
||||
t *testing.T,
|
||||
) (*httptest.Server, *store.MemoryStore, *httpFakeObjectStore) {
|
||||
t.Helper()
|
||||
memoryStore := store.NewMemoryStore()
|
||||
repository := store.NewMemoryRepository(memoryStore)
|
||||
objectStore := &httpFakeObjectStore{objects: make(map[string][]byte)}
|
||||
service := newTestService(
|
||||
repository,
|
||||
time.Hour,
|
||||
application.WithObjectStore(objectStore),
|
||||
)
|
||||
handler := handlers.NewHandler(service)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
requireLegacyAuth(memoryStore, handler.UploadAttachment),
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"GET /api/v1/attachments/{attachment_id}",
|
||||
requireLegacyAuth(memoryStore, handler.GetAttachment),
|
||||
)
|
||||
mux.HandleFunc(
|
||||
"GET /api/v1/attachments/{attachment_id}/content",
|
||||
requireLegacyAuth(memoryStore, handler.DownloadAttachment),
|
||||
)
|
||||
var root http.Handler = mux
|
||||
root = middleware.Recovery(root)
|
||||
root = middleware.RequestID(root)
|
||||
return httptest.NewServer(root), memoryStore, objectStore
|
||||
}
|
||||
|
||||
func pngPayload(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var payload bytes.Buffer
|
||||
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
|
||||
img.Set(0, 0, color.RGBA{R: 255, A: 255})
|
||||
if err := png.Encode(&payload, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return payload.Bytes()
|
||||
}
|
||||
|
||||
func uploadRequest(
|
||||
t *testing.T,
|
||||
baseURL string,
|
||||
chatID string,
|
||||
userID string,
|
||||
filename string,
|
||||
contentType string,
|
||||
payload []byte,
|
||||
) *http.Response {
|
||||
t.Helper()
|
||||
requestURL := baseURL + "/api/v1/chats/" + chatID +
|
||||
"/attachments?filename=" + url.QueryEscape(filename)
|
||||
request, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
request.Header.Set("X-User-Id", userID)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func attachmentGetRequest(t *testing.T, requestURL string, userID string) *http.Response {
|
||||
t.Helper()
|
||||
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("X-User-Id", userID)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func TestUploadAttachmentHTTPContract(t *testing.T) {
|
||||
server, memoryStore, objectStore := setupAttachmentUploadServer(t)
|
||||
defer server.Close()
|
||||
user, err := memoryStore.CreateUser("attachment_user", "Attachment User")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outsider, err := memoryStore.CreateUser("attachment_outsider", "Attachment Outsider")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chat, err := memoryStore.CreateChat("Images", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := pngPayload(t)
|
||||
response := uploadRequest(t, server.URL, chat.ID, user.ID, "кот.png", "image/png", payload)
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
t.Fatalf("status = %d, want 201: %s", response.StatusCode, body)
|
||||
}
|
||||
var attachment dto.AttachmentResponse
|
||||
if err = json.NewDecoder(response.Body).Decode(&attachment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if attachment.Status != domain.AttachmentStatusReady || attachment.Filename != "кот.png" {
|
||||
t.Fatalf("unexpected response: %+v", attachment)
|
||||
}
|
||||
if attachment.Variants == nil || len(attachment.Variants) != 0 || attachment.ErrorCode != nil {
|
||||
t.Fatalf("contract arrays/nulls differ: %+v", attachment)
|
||||
}
|
||||
objectStore.mu.Lock()
|
||||
storedObjects := len(objectStore.objects)
|
||||
objectStore.mu.Unlock()
|
||||
if storedObjects != 1 {
|
||||
t.Fatalf("stored objects = %d, want 1", storedObjects)
|
||||
}
|
||||
|
||||
metadataResponse := attachmentGetRequest(t, server.URL+"/api/v1/attachments/"+attachment.ID, user.ID)
|
||||
defer metadataResponse.Body.Close()
|
||||
if metadataResponse.StatusCode != http.StatusOK {
|
||||
t.Fatalf("metadata status = %d, want 200", metadataResponse.StatusCode)
|
||||
}
|
||||
|
||||
contentResponse := attachmentGetRequest(t,
|
||||
server.URL+"/api/v1/attachments/"+attachment.ID+"/content", user.ID)
|
||||
defer contentResponse.Body.Close()
|
||||
downloaded, err := io.ReadAll(contentResponse.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contentResponse.StatusCode != http.StatusOK || !bytes.Equal(downloaded, payload) {
|
||||
t.Fatalf("download status/content differ: status=%d", contentResponse.StatusCode)
|
||||
}
|
||||
if contentResponse.Header.Get("Content-Type") != "image/png" {
|
||||
t.Fatalf("download Content-Type = %q", contentResponse.Header.Get("Content-Type"))
|
||||
}
|
||||
|
||||
for _, suffix := range []string{"", "/content"} {
|
||||
outsiderResponse := attachmentGetRequest(t,
|
||||
server.URL+"/api/v1/attachments/"+attachment.ID+suffix, outsider.ID)
|
||||
_ = outsiderResponse.Body.Close()
|
||||
if outsiderResponse.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("outsider %q status = %d, want 404", suffix, outsiderResponse.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
missingVariant := attachmentGetRequest(t,
|
||||
server.URL+"/api/v1/attachments/"+attachment.ID+"/content?variant=thumbnail", user.ID)
|
||||
_ = missingVariant.Body.Close()
|
||||
if missingVariant.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("missing variant status = %d, want 404", missingVariant.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAttachmentHTTPErrorMapping(t *testing.T) {
|
||||
server, memoryStore, _ := setupAttachmentUploadServer(t)
|
||||
defer server.Close()
|
||||
user, _ := memoryStore.CreateUser("attachment_errors", "Attachment Errors")
|
||||
chat, _ := memoryStore.CreateChat("Images", []string{user.ID})
|
||||
validPNG := pngPayload(t)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
chatID string
|
||||
filename string
|
||||
contentType string
|
||||
payload []byte
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "unsafe filename", chatID: chat.ID, filename: "../image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusBadRequest},
|
||||
{name: "unsupported content type", chatID: chat.ID, filename: "image.gif", contentType: "image/gif", payload: []byte("GIF89a"), wantStatus: http.StatusUnsupportedMediaType},
|
||||
{name: "MIME substitution", chatID: chat.ID, filename: "image.jpg", contentType: "image/jpeg", payload: validPNG, wantStatus: http.StatusUnsupportedMediaType},
|
||||
{name: "corrupt image", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: []byte("bad"), wantStatus: http.StatusUnsupportedMediaType},
|
||||
{name: "too large", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantStatus: http.StatusRequestEntityTooLarge},
|
||||
{name: "unknown chat", chatID: "11111111-1111-4111-8111-111111111111", filename: "image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusNotFound},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
response := uploadRequest(t, server.URL, testCase.chatID, user.ID,
|
||||
testCase.filename, testCase.contentType, testCase.payload)
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != testCase.wantStatus {
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
t.Fatalf("status = %d, want %d: %s", response.StatusCode, testCase.wantStatus, body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
@@ -133,7 +134,7 @@ func TestMessageIdempotencyReplayAndConflict(t *testing.T) {
|
||||
|
||||
first := decodeMessageResponse(t, firstPayload)
|
||||
replay := decodeMessageResponse(t, replayPayload)
|
||||
if first != replay {
|
||||
if !reflect.DeepEqual(first, replay) {
|
||||
t.Fatalf("replay changed message: first=%+v replay=%+v", first, replay)
|
||||
}
|
||||
|
||||
@@ -256,7 +257,7 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) {
|
||||
for message := range results {
|
||||
if count == 0 {
|
||||
expected = message
|
||||
} else if message != expected {
|
||||
} else if !reflect.DeepEqual(message, expected) {
|
||||
t.Fatalf("concurrent replay returned different message: %+v != %+v", message, expected)
|
||||
}
|
||||
count++
|
||||
@@ -273,3 +274,40 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) {
|
||||
t.Fatalf("expected exactly one stored message, got %d", len(messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageIdempotencyTreatsMissingAndEmptyAttachmentsEqually(t *testing.T) {
|
||||
_, serverURL, user, chatID := createMessageFixture(t)
|
||||
|
||||
status, firstPayload := sendMessageRequest(
|
||||
t,
|
||||
serverURL,
|
||||
chatID,
|
||||
user.ID,
|
||||
"default-attachments",
|
||||
`{"text":"same semantic request"}`,
|
||||
)
|
||||
if status != http.StatusCreated {
|
||||
t.Fatalf("first request: expected 201, got %d: %s", status, firstPayload)
|
||||
}
|
||||
|
||||
status, replayPayload := sendMessageRequest(
|
||||
t,
|
||||
serverURL,
|
||||
chatID,
|
||||
user.ID,
|
||||
"default-attachments",
|
||||
`{"attachment_ids":[],"text":"same semantic request"}`,
|
||||
)
|
||||
if status != http.StatusCreated {
|
||||
t.Fatalf("replay: expected 201, got %d: %s", status, replayPayload)
|
||||
}
|
||||
|
||||
first := decodeMessageResponse(t, firstPayload)
|
||||
replay := decodeMessageResponse(t, replayPayload)
|
||||
if !reflect.DeepEqual(first, replay) {
|
||||
t.Fatalf("missing attachment_ids and [] must replay identically")
|
||||
}
|
||||
if first.AttachmentIDs == nil || len(first.AttachmentIDs) != 0 {
|
||||
t.Fatalf("response must contain attachment_ids: [], got %#v", first.AttachmentIDs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package student_tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"mtusi2026lab4/internal/domain"
|
||||
)
|
||||
|
||||
func TestMessageAttachmentContract(t *testing.T) {
|
||||
memoryStore, serverURL, user, chatID := createMessageFixture(t)
|
||||
|
||||
readyID := "11111111-1111-4111-8111-111111111111"
|
||||
if err := memoryStore.PutAttachment(domain.Attachment{
|
||||
ID: readyID,
|
||||
ChatID: chatID,
|
||||
OwnerID: user.ID,
|
||||
Status: domain.AttachmentStatusReady,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed ready attachment: %v", err)
|
||||
}
|
||||
|
||||
status, payload := sendMessageRequest(
|
||||
t,
|
||||
serverURL,
|
||||
chatID,
|
||||
user.ID,
|
||||
"attachment-only",
|
||||
fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, readyID),
|
||||
)
|
||||
if status != http.StatusCreated {
|
||||
t.Fatalf("attachment-only message: expected 201, got %d: %s", status, payload)
|
||||
}
|
||||
|
||||
message := decodeMessageResponse(t, payload)
|
||||
if message.Text != "" {
|
||||
t.Fatalf("attachment-only message changed text: %q", message.Text)
|
||||
}
|
||||
if !reflect.DeepEqual(message.AttachmentIDs, []string{readyID}) {
|
||||
t.Fatalf("attachment_ids changed: %#v", message.AttachmentIDs)
|
||||
}
|
||||
|
||||
items, _, err := memoryStore.GetMessages(chatID, user.ID, 10, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("read stored messages: %v", err)
|
||||
}
|
||||
if len(items) != 1 || !reflect.DeepEqual(items[0].AttachmentIDs, []string{readyID}) {
|
||||
t.Fatalf("stored attachment relation differs: %#v", items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageAttachmentValidationAndAccess(t *testing.T) {
|
||||
memoryStore, serverURL, user, chatID := createMessageFixture(t)
|
||||
otherChat, err := memoryStore.CreateChat("Other chat", []string{user.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("create other chat: %v", err)
|
||||
}
|
||||
|
||||
pendingID := "22222222-2222-4222-8222-222222222222"
|
||||
if err = memoryStore.PutAttachment(domain.Attachment{
|
||||
ID: pendingID,
|
||||
ChatID: chatID,
|
||||
OwnerID: user.ID,
|
||||
Status: domain.AttachmentStatusPending,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed pending attachment: %v", err)
|
||||
}
|
||||
|
||||
foreignID := "33333333-3333-4333-8333-333333333333"
|
||||
if err = memoryStore.PutAttachment(domain.Attachment{
|
||||
ID: foreignID,
|
||||
ChatID: otherChat.ID,
|
||||
OwnerID: user.ID,
|
||||
Status: domain.AttachmentStatusReady,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed foreign attachment: %v", err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
body string
|
||||
wantStatus int
|
||||
}{
|
||||
{
|
||||
name: "empty without attachments",
|
||||
body: `{"text":"","attachment_ids":[]}`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "invalid attachment uuid",
|
||||
body: `{"text":"hello","attachment_ids":["not-a-uuid"]}`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "duplicate attachment",
|
||||
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q,%q]}`, pendingID, pendingID),
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "unknown attachment",
|
||||
body: `{"text":"hello","attachment_ids":["44444444-4444-4444-8444-444444444444"]}`,
|
||||
wantStatus: http.StatusNotFound,
|
||||
},
|
||||
{
|
||||
name: "attachment from another chat",
|
||||
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, foreignID),
|
||||
wantStatus: http.StatusNotFound,
|
||||
},
|
||||
{
|
||||
name: "attachment not ready",
|
||||
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, pendingID),
|
||||
wantStatus: http.StatusConflict,
|
||||
},
|
||||
}
|
||||
|
||||
for index, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
status, payload := sendMessageRequest(
|
||||
t,
|
||||
serverURL,
|
||||
chatID,
|
||||
user.ID,
|
||||
fmt.Sprintf("attachment-validation-%d", index),
|
||||
testCase.body,
|
||||
)
|
||||
if status != testCase.wantStatus {
|
||||
t.Fatalf("expected %d, got %d: %s", testCase.wantStatus, status, payload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Black-box attachment validation and exact contract-boundary checks."""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from attachment_memory import cleanup, noisy_png, png_chunk
|
||||
from client import Client, check, run
|
||||
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def exact_size_png(sample, target_size=MAX_BYTES):
|
||||
check(sample.endswith(png_chunk(b"IEND", b"")), "sample PNG has no final IEND")
|
||||
prefix, iend = sample[:-12], sample[-12:]
|
||||
filler_size = target_size - len(prefix) - len(iend) - 12
|
||||
check(filler_size >= 0, "sample PNG is larger than boundary")
|
||||
result = prefix + png_chunk(b"ruSt", bytes(filler_size)) + iend
|
||||
check(len(result) == target_size, "exact-size PNG construction failed")
|
||||
return result
|
||||
|
||||
|
||||
def exact_pixel_png(width=10_000, height=4_000):
|
||||
compressor = zlib.compressobj(level=9)
|
||||
row = bytes(1 + width * 3)
|
||||
parts = []
|
||||
for _ in range(height):
|
||||
chunk = compressor.compress(row)
|
||||
if chunk:
|
||||
parts.append(chunk)
|
||||
parts.append(compressor.flush())
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
+ png_chunk(b"IDAT", b"".join(parts))
|
||||
+ png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def config_only_png(width, height):
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
)
|
||||
|
||||
|
||||
def upload(client, actor, chat, filename, payload, expected, schema=None):
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename),
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(os.environ.get("BASE_URL"), None, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
users = [alice["id"], bob["id"], eve["id"]]
|
||||
sample = open(os.path.join(os.path.dirname(__file__), "fixtures", "sample.png"), "rb").read()
|
||||
|
||||
try:
|
||||
boundary = exact_size_png(sample)
|
||||
response = upload(client, alice, chat, "exact-10mib.png", boundary, 201, "Attachment")
|
||||
attachment = response["json"]
|
||||
check(attachment["size_bytes"] == MAX_BYTES, "10 MiB boundary size changed")
|
||||
downloaded = client.call(
|
||||
"GET", "/api/v1/attachments/%s/content" % attachment["id"],
|
||||
200, actor=alice,
|
||||
)["bytes"]
|
||||
check(downloaded == boundary, "10 MiB original changed after round trip")
|
||||
|
||||
pixels = exact_pixel_png()
|
||||
upload(client, alice, chat, "exact-40m-pixels.png", pixels, 201, "Attachment")
|
||||
|
||||
upload(client, alice, chat, "over-pixels.png", config_only_png(20_000, 10_000), 413)
|
||||
upload(client, alice, chat, "truncated.png", sample[:33], 415)
|
||||
|
||||
corrupted = noisy_png()
|
||||
check(len(corrupted) > 70_000, "corruption fixture is too small")
|
||||
corrupted = bytearray(corrupted)
|
||||
corrupted[70_000] ^= 0xff
|
||||
upload(client, alice, chat, "corrupt-after-prefix.png", bytes(corrupted), 415)
|
||||
|
||||
upload(client, alice, chat, "%20%20%20", sample, 400)
|
||||
upload(client, alice, chat, "c1%C2%85.png", sample, 400)
|
||||
upload(client, alice, chat, "invalid-%FF.png", sample, 400)
|
||||
print("PASS: exact byte/pixel boundaries, complete decode, and filename validation are correct.")
|
||||
finally:
|
||||
cleanup(chat["id"], users)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent real PNG uploads with Docker API-container memory sampling."""
|
||||
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
from client import Client, check, run
|
||||
|
||||
|
||||
def png_chunk(kind, payload):
|
||||
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload))
|
||||
|
||||
|
||||
def noisy_png(width=2048, height=1024):
|
||||
rows = bytearray()
|
||||
for _ in range(height):
|
||||
rows.append(0)
|
||||
rows.extend(os.urandom(width * 3))
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
+ png_chunk(b"IDAT", zlib.compress(bytes(rows), level=0))
|
||||
+ png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def bytes_from_docker(value):
|
||||
number, unit = value.split()[0], value.split()[0][-3:]
|
||||
if unit in {"KiB", "MiB", "GiB"}:
|
||||
number = value.split()[0][:-3]
|
||||
else:
|
||||
unit = "B"
|
||||
factor = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}[unit]
|
||||
return int(float(number) * factor)
|
||||
|
||||
|
||||
def api_container_ids():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "-q", "api"],
|
||||
check=True, text=True, capture_output=True,
|
||||
)
|
||||
return [line for line in result.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def sample_memory(container_ids, stop_event, peaks):
|
||||
while not stop_event.is_set():
|
||||
result = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}", *container_ids],
|
||||
check=False, text=True, capture_output=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
for line in result.stdout.splitlines():
|
||||
name, usage = line.split("\t", 1)
|
||||
current = bytes_from_docker(usage.split(" / ", 1)[0])
|
||||
peaks[name] = max(peaks.get(name, 0), current)
|
||||
stop_event.wait(0.05)
|
||||
|
||||
|
||||
def cleanup(chat_id, user_ids):
|
||||
object_script = (
|
||||
'root_user="$(cat /run/secrets/minio_root_user)"\n'
|
||||
'root_password="$(cat /run/secrets/minio_root_password)"\n'
|
||||
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
|
||||
'mc rm --recursive --force "local/${S3_BUCKET}/attachments/' + chat_id + '/" >/dev/null 2>&1 || true\n'
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "compose", "run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", object_script],
|
||||
check=True,
|
||||
)
|
||||
quoted_users = ",".join("'%s'" % value for value in user_ids)
|
||||
sql = f"""
|
||||
BEGIN;
|
||||
DELETE FROM messages WHERE chat_id = '{chat_id}';
|
||||
DELETE FROM attachments WHERE chat_id = '{chat_id}';
|
||||
DELETE FROM sessions WHERE user_id IN ({quoted_users});
|
||||
DELETE FROM chat_members WHERE chat_id = '{chat_id}';
|
||||
DELETE FROM chats WHERE id = '{chat_id}';
|
||||
DELETE FROM users WHERE id IN ({quoted_users});
|
||||
COMMIT;
|
||||
"""
|
||||
subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger"],
|
||||
input=sql, text=True, check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
concurrency = 12
|
||||
base_url = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
client = Client(base_url, None, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
users = [alice["id"], bob["id"], eve["id"]]
|
||||
payload = noisy_png()
|
||||
stop_event = threading.Event()
|
||||
peaks = {}
|
||||
sampler = threading.Thread(
|
||||
target=sample_memory,
|
||||
args=(api_container_ids(), stop_event, peaks),
|
||||
daemon=True,
|
||||
)
|
||||
sampler.start()
|
||||
started = time.monotonic()
|
||||
|
||||
def upload(index):
|
||||
response = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=memory-%02d.png" % (chat["id"], index),
|
||||
201,
|
||||
actor=alice,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png", "X-Load-Id": uuid.uuid4().hex},
|
||||
schema="Attachment",
|
||||
)
|
||||
return response["headers"]["X-Instance-Id"]
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
instances = list(pool.map(upload, range(concurrency)))
|
||||
elapsed = time.monotonic() - started
|
||||
stop_event.set()
|
||||
sampler.join(timeout=10)
|
||||
check(len(set(instances)) >= 2, "uploads did not exercise both API replicas")
|
||||
check(len(peaks) >= 2, "memory was not sampled for both API replicas")
|
||||
summary = {
|
||||
"concurrency": concurrency,
|
||||
"payload_bytes": len(payload),
|
||||
"total_uploaded_bytes": len(payload) * concurrency,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_uploads": len(instances),
|
||||
"api_instances_observed": len(set(instances)),
|
||||
"peak_memory_bytes_by_container": peaks,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
||||
print("PASS: concurrent attachment uploads remained within configured API memory limits.")
|
||||
finally:
|
||||
stop_event.set()
|
||||
sampler.join(timeout=10)
|
||||
cleanup(chat["id"], users)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_root"
|
||||
|
||||
api_containers=$(docker compose ps -q api)
|
||||
if [ -z "$api_containers" ]; then
|
||||
printf "%s\n" "no running API containers found" >&2
|
||||
exit 1
|
||||
fi
|
||||
for container_id in $api_containers; do
|
||||
docker exec "$container_id" sh -ec '
|
||||
test ! -e /run/secrets/minio_root_user
|
||||
test ! -e /run/secrets/minio_root_password
|
||||
! env | grep -Eq "^MINIO_ROOT_(USER|PASSWORD)="
|
||||
'
|
||||
done
|
||||
|
||||
docker compose run --rm --entrypoint /bin/sh minio-bootstrap -ec '
|
||||
root_user="$(cat /run/secrets/minio_root_user)"
|
||||
root_password="$(cat /run/secrets/minio_root_password)"
|
||||
api_user="$(cat /run/secrets/s3_access_key_id)"
|
||||
api_password="$(cat /run/secrets/s3_secret_access_key)"
|
||||
probe_key="policy-probe/object.txt"
|
||||
|
||||
mc alias set admin http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null
|
||||
mc alias set limited http://minio:9000 "$api_user" "$api_password" --api S3v4 >/dev/null
|
||||
|
||||
cleanup() {
|
||||
mc rm --force "admin/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1 || true
|
||||
mc rb --force "admin/forbidden-policy-probe" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
printf private-probe | mc pipe "limited/${S3_BUCKET}/${probe_key}" >/dev/null
|
||||
mc stat "limited/${S3_BUCKET}/${probe_key}" >/dev/null
|
||||
mc cat "limited/${S3_BUCKET}/${probe_key}" >/dev/null
|
||||
|
||||
if mc mb "limited/forbidden-policy-probe" >/dev/null 2>&1; then
|
||||
printf "%s\n" "limited API credentials unexpectedly created another bucket" >&2
|
||||
exit 1
|
||||
fi
|
||||
if wget -qO- "http://minio:9000/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1; then
|
||||
printf "%s\n" "anonymous HTTP GET unexpectedly read a private object" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc rm --force "limited/${S3_BUCKET}/${probe_key}" >/dev/null
|
||||
trap - EXIT INT TERM
|
||||
printf "%s\n" "PASS: API has no MinIO root credentials; limited policy permits object IO only in the private bucket; anonymous GET is denied."
|
||||
'
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inject isolated DB/S3 inconsistencies and prove reconciler dry-run/apply/repeat."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
USER_ID = "90000000-0000-4000-8000-000000000001"
|
||||
CHAT_ID = "90000000-0000-4000-8000-000000000002"
|
||||
MISSING_ID = "90000000-0000-4000-8000-000000000011"
|
||||
PROTECTED_ID = "90000000-0000-4000-8000-000000000012"
|
||||
PENDING_ID = "90000000-0000-4000-8000-000000000013"
|
||||
MESSAGE_ID = "90000000-0000-4000-8000-000000000021"
|
||||
PREFIX = "attachments/reconcile-integration"
|
||||
ORPHAN_KEY = PREFIX + "/orphan"
|
||||
PENDING_KEY = PREFIX + "/pending"
|
||||
|
||||
|
||||
def run(command, *, input_text=None, capture=False):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
input=input_text,
|
||||
text=True,
|
||||
check=False,
|
||||
capture_output=capture,
|
||||
env=os.environ.copy(),
|
||||
)
|
||||
if result.returncode != 0:
|
||||
details = result.stderr.strip() if capture else "see command output"
|
||||
raise RuntimeError("command failed: %s: %s" % (command[0], details))
|
||||
return result.stdout if capture else ""
|
||||
|
||||
|
||||
def compose(*arguments, capture=False, input_text=None):
|
||||
return run(
|
||||
["docker", "compose", *arguments],
|
||||
input_text=input_text,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
|
||||
def database(sql):
|
||||
compose(
|
||||
"exec", "-T", "postgres",
|
||||
"psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger",
|
||||
input_text=sql,
|
||||
)
|
||||
|
||||
|
||||
def object_command(script):
|
||||
compose(
|
||||
"run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap",
|
||||
"-ec", script,
|
||||
)
|
||||
|
||||
|
||||
def clean_objects():
|
||||
object_command(
|
||||
'root_user="$(cat /run/secrets/minio_root_user)"\n'
|
||||
'root_password="$(cat /run/secrets/minio_root_password)"\n'
|
||||
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
|
||||
'mc rm --force "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null 2>&1 || true\n'
|
||||
'mc rm --force "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null 2>&1 || true\n'
|
||||
)
|
||||
|
||||
|
||||
def clean_database():
|
||||
database(f"""
|
||||
BEGIN;
|
||||
DELETE FROM messages WHERE id = '{MESSAGE_ID}';
|
||||
DELETE FROM attachments WHERE id IN ('{MISSING_ID}', '{PROTECTED_ID}', '{PENDING_ID}');
|
||||
DELETE FROM chat_members WHERE chat_id = '{CHAT_ID}';
|
||||
DELETE FROM chats WHERE id = '{CHAT_ID}';
|
||||
DELETE FROM users WHERE id = '{USER_ID}';
|
||||
COMMIT;
|
||||
""")
|
||||
|
||||
|
||||
def inject_database():
|
||||
database(f"""
|
||||
BEGIN;
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('{USER_ID}', 'reconcile_fixture', 'Reconcile Fixture', 'not-used');
|
||||
INSERT INTO chats (id, title) VALUES ('{CHAT_ID}', 'Reconcile Integration');
|
||||
INSERT INTO chat_members (chat_id, user_id, position) VALUES ('{CHAT_ID}', '{USER_ID}', 0);
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, object_key, filename, content_type,
|
||||
size_bytes, sha256, width, height, status, created_at, updated_at
|
||||
) VALUES
|
||||
('{MISSING_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/missing', 'missing.png', 'image/png',
|
||||
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'),
|
||||
('{PROTECTED_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/protected-missing', 'protected.png', 'image/png',
|
||||
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'),
|
||||
('{PENDING_ID}', '{CHAT_ID}', '{USER_ID}', '{PENDING_KEY}', 'pending.png', 'image/png',
|
||||
7, decode(repeat('00', 32), 'hex'), 1, 1, 'pending', now() - interval '2 hours', now() - interval '2 hours');
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{USER_ID}', 'protected attachment reference');
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{PROTECTED_ID}', 0);
|
||||
COMMIT;
|
||||
""")
|
||||
|
||||
|
||||
def inject_objects():
|
||||
object_command(
|
||||
'root_user="$(cat /run/secrets/minio_root_user)"\n'
|
||||
'root_password="$(cat /run/secrets/minio_root_password)"\n'
|
||||
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
|
||||
'printf orphan | mc pipe "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null\n'
|
||||
'printf pending | mc pipe "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null\n'
|
||||
)
|
||||
|
||||
|
||||
def reconcile(apply=False):
|
||||
command = [
|
||||
"run", "--rm", "--entrypoint", "/app/reconciler", "api",
|
||||
"--grace-period", "1s",
|
||||
]
|
||||
if apply:
|
||||
command.append("--apply")
|
||||
output = compose(*command, capture=True)
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def kinds(result):
|
||||
return {issue["kind"] for issue in result["issues"]}
|
||||
|
||||
|
||||
def main():
|
||||
clean_objects()
|
||||
clean_database()
|
||||
try:
|
||||
inject_database()
|
||||
inject_objects()
|
||||
time.sleep(2)
|
||||
|
||||
dry_run = reconcile(False)
|
||||
expected = {
|
||||
"orphan_object", "missing_object", "referenced_missing_object", "stale_pending",
|
||||
}
|
||||
if not expected.issubset(kinds(dry_run)):
|
||||
raise RuntimeError("dry-run did not detect all injected inconsistencies")
|
||||
if dry_run["deleted_objects"] or dry_run["failed_attachments"]:
|
||||
raise RuntimeError("dry-run performed mutations")
|
||||
|
||||
applied = reconcile(True)
|
||||
if applied["deleted_objects"] != 2 or applied["failed_attachments"] != 2:
|
||||
raise RuntimeError("apply mutation counts differ: %s" % applied)
|
||||
|
||||
repeated = reconcile(True)
|
||||
if repeated["deleted_objects"] or repeated["failed_attachments"]:
|
||||
raise RuntimeError("repeated apply was not mutation-idempotent")
|
||||
protected = [issue for issue in repeated["issues"] if issue.get("protected")]
|
||||
if len(protected) != 1 or protected[0]["action"] != "manual_repair_required":
|
||||
raise RuntimeError("referenced missing attachment was not protected")
|
||||
|
||||
print(json.dumps({
|
||||
"dry_run_issues": len(dry_run["issues"]),
|
||||
"apply_deleted_objects": applied["deleted_objects"],
|
||||
"apply_failed_attachments": applied["failed_attachments"],
|
||||
"repeat_mutations": repeated["deleted_objects"] + repeated["failed_attachments"],
|
||||
"protected_manual_issues": len(protected),
|
||||
"scanned_attachments": applied["scanned_attachments"],
|
||||
"scanned_objects": applied["scanned_objects"],
|
||||
"duration_ms": applied["duration_ms"],
|
||||
}, ensure_ascii=False))
|
||||
print("PASS: real PostgreSQL/MinIO reconciliation is dry-run safe, idempotent and reference-aware.")
|
||||
finally:
|
||||
clean_objects()
|
||||
clean_database()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure real PostgreSQL/MinIO reconciler traversal at two inventory sizes."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
PREFIX = "attachments/reconcile-scale"
|
||||
|
||||
|
||||
def compose(*arguments, capture=False):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", *arguments],
|
||||
check=False,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(result.stderr.strip() if capture else "docker compose command failed")
|
||||
return result.stdout if capture else ""
|
||||
|
||||
|
||||
def object_command(script):
|
||||
compose("run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", script)
|
||||
|
||||
|
||||
def clean():
|
||||
object_command(
|
||||
'root_user="$(cat /run/secrets/minio_root_user)"\n'
|
||||
'root_password="$(cat /run/secrets/minio_root_password)"\n'
|
||||
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
|
||||
'mc rm --recursive --force "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null 2>&1 || true\n'
|
||||
)
|
||||
|
||||
|
||||
def create(count):
|
||||
object_command(
|
||||
'root_user="$(cat /run/secrets/minio_root_user)"\n'
|
||||
'root_password="$(cat /run/secrets/minio_root_password)"\n'
|
||||
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
|
||||
'mkdir -p /tmp/reconcile-scale\n'
|
||||
'index=0\n'
|
||||
'while [ "$index" -lt ' + str(count) + ' ]; do\n'
|
||||
' printf x > "/tmp/reconcile-scale/object-${index}"\n'
|
||||
' index=$((index + 1))\n'
|
||||
'done\n'
|
||||
'mc cp --recursive /tmp/reconcile-scale/ "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null\n'
|
||||
)
|
||||
|
||||
|
||||
def dry_run():
|
||||
started = time.monotonic()
|
||||
output = compose(
|
||||
"run", "--rm", "--entrypoint", "/app/reconciler", "api",
|
||||
"--grace-period", "1s",
|
||||
capture=True,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
result = json.loads(output)
|
||||
return {
|
||||
"scanned_attachments": result["scanned_attachments"],
|
||||
"scanned_objects": result["scanned_objects"],
|
||||
"issues": len(result["issues"]),
|
||||
"reconciler_duration_ms": result["duration_ms"],
|
||||
"container_start_and_run_ms": round(elapsed * 1000),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
measurements = []
|
||||
clean()
|
||||
try:
|
||||
for count in (100, 1000):
|
||||
clean()
|
||||
create(count)
|
||||
time.sleep(2)
|
||||
measurement = dry_run()
|
||||
measurement["injected_orphan_objects"] = count
|
||||
if measurement["issues"] < count:
|
||||
raise RuntimeError("reconciler did not observe all scale fixtures")
|
||||
measurements.append(measurement)
|
||||
print(json.dumps({"measurements": measurements}, ensure_ascii=False))
|
||||
print("PASS: reconciler traversal cost measured on real PostgreSQL/MinIO inventories.")
|
||||
finally:
|
||||
clean()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify contract responses and original durability during a real MinIO outage."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from attachment_memory import cleanup
|
||||
from client import Client, Failure, check, run
|
||||
|
||||
|
||||
def compose(*arguments, **kwargs):
|
||||
return subprocess.run(["docker", "compose", *arguments], check=True, **kwargs)
|
||||
|
||||
|
||||
def wait_for_status(client, path, expected, timeout=20):
|
||||
deadline = time.monotonic() + timeout
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
return client.call("GET", path, expected=expected)
|
||||
except Failure as error:
|
||||
last_error = error
|
||||
time.sleep(0.25)
|
||||
raise Failure("%s did not reach %d: %s" % (path, expected, last_error))
|
||||
|
||||
|
||||
def restore_storage(client):
|
||||
compose("up", "-d", "--wait", "minio")
|
||||
compose("run", "--rm", "minio-bootstrap")
|
||||
wait_for_status(client, "/health/ready", 200, timeout=30)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(os.environ.get("BASE_URL"), None, timeout=15)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
users = [alice["id"], bob["id"], eve["id"]]
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
storage_stopped = False
|
||||
|
||||
try:
|
||||
compose("stop", "minio")
|
||||
storage_stopped = True
|
||||
readiness = wait_for_status(client, "/health/ready", 503)
|
||||
check(readiness["elapsed"] < 5, "readiness exceeded its bounded timeout")
|
||||
|
||||
started = time.monotonic()
|
||||
failure = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=outage.png" % chat["id"],
|
||||
503,
|
||||
actor=alice,
|
||||
raw=original,
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
check(failure["json"]["error"]["code"] == "unavailable", "wrong S3 outage error code")
|
||||
check(elapsed < 15, "upload outage response exceeded API/client timeout")
|
||||
|
||||
restore_storage(client)
|
||||
storage_stopped = False
|
||||
downloaded = client.call(
|
||||
"GET", "/api/v1/attachments/%s/content" % attachment["id"],
|
||||
200, actor=alice,
|
||||
)["bytes"]
|
||||
check(downloaded == original, "existing original changed across MinIO outage")
|
||||
print(json.dumps({
|
||||
"readiness_503_seconds": round(readiness["elapsed"], 3),
|
||||
"upload_503_seconds": round(elapsed, 3),
|
||||
"original_bytes_verified": len(original),
|
||||
}, sort_keys=True))
|
||||
print("PASS: MinIO outage returns bounded JSON 503 and preserves existing originals.")
|
||||
finally:
|
||||
if storage_stopped:
|
||||
restore_storage(client)
|
||||
cleanup(chat["id"], users)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user