лаба 4 на оценку 5

This commit is contained in:
2026-10-30 12:27:49 +03:00
parent 7860d81fa1
commit f9444a3965
59 changed files with 4696 additions and 90 deletions
+6 -2
View File
@@ -1,7 +1,8 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_INTERNAL_PORT=8080
PUBLIC_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
@@ -11,10 +12,13 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_OPERATION_TIMEOUT=5s
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Keep the Go soft heap limit below the Compose hard limit (256 MiB by default).
API_GO_MEMORY_LIMIT=220MiB
+3 -1
View File
@@ -10,6 +10,7 @@ COPY internal/ ./internal/
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/server ./cmd/server/main.go
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-w -s" -o /build/reconciler ./cmd/reconciler/main.go
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
@@ -19,6 +20,7 @@ RUN addgroup -g 1000 appgroup && \
WORKDIR /app
COPY --from=builder /build/server /app/server
COPY --from=builder /build/reconciler /app/reconciler
RUN chown -R appuser:appgroup /app
@@ -30,4 +32,4 @@ HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=10 \
CMD wget -qO- http://127.0.0.1:8080/health/ready || exit 1
ENTRYPOINT ["/app/server"]
ENTRYPOINT ["/app/server"]
+29 -1
View File
@@ -2,13 +2,17 @@ PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
RECONCILE_GRACE ?= 15m
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help init-secrets check validate test load test-attachment-boundaries test-attachment-memory test-storage-outage test-minio-policy test-persistence test-failover test-reconcile test-reconcile-scale reconcile-dry-run reconcile-apply
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
init-secrets:
./scripts/init-secrets.sh
check:
$(PYTHON) scripts/check_contract.py
@@ -21,6 +25,18 @@ test:
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachment-memory:
BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_memory.py
test-attachment-boundaries:
BASE_URL="$(BASE_URL)" $(PYTHON) tests/attachment_boundaries.py
test-storage-outage:
BASE_URL="$(BASE_URL)" $(PYTHON) tests/storage_outage.py
test-minio-policy:
./tests/minio_policy.sh
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +44,15 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile:
$(PYTHON) tests/reconcile_integration.py
test-reconcile-scale:
$(PYTHON) tests/reconcile_scale.py
reconcile-dry-run:
docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)"
reconcile-apply:
docker compose run --rm --entrypoint /app/reconciler api --grace-period "$(RECONCILE_GRACE)" --apply
+103 -17
View File
@@ -1,37 +1,123 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: TODO — заполнить перед сдачей.
- Целевая оценка: **5**, выбран **трек 2 — согласованность PostgreSQL и S3**.
- Commit/PR: TODO после публикации.
- Использованные источники и библиотеки: Go standard library, `pgx/v5`, `minio-go/v7`, официальный исходный код MinIO и `mc`, Redis Go client. OpenAI Codex использовался для анализа контракта, проектирования, реализации и тестов; все утверждения ниже проверены перечисленными воспроизводимыми командами.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker с Compose v2, GNU Make, Python 3.9+, OpenSSL. Локально проверено в WSL2. MinIO и `mc` собираются из официальных Go-модулей в [infrastructure/minio/Dockerfile](infrastructure/minio/Dockerfile): сервер `RELEASE.2025-10-15T17-29-55Z`, клиент `RELEASE.2025-08-13T08-35-41Z`. Базовые образы закреплены digest.
```sh
cp .env.example .env
make init-secrets
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
docker compose --env-file .env ps
BASE_URL=http://127.0.0.1:8080 make test
python3 -m venv .venv
.venv/bin/pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
```
`make init-secrets` создаёт недостающие файлы с mode `600` и никогда не перезаписывает существующие значения. Секреты находятся в игнорируемом каталоге `secrets/` и не передаются через переменные окружения контейнера. Если `8080` занят, задаётся `PUBLIC_PORT`, например `PUBLIC_PORT=18080`; внутренний `APP_INTERNAL_PORT=8080` от него не зависит.
Безопасная остановка сохраняет named volumes:
```sh
docker compose --env-file .env down
```
Намеренное удаление всех учебных данных выполняется только явно:
```sh
docker compose --env-file .env down -v
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Путь запроса: клиент → HAProxy → две API-реплики → PostgreSQL (источник истины метаданных/сессий) + приватный MinIO bucket (байты); Redis остаётся восстанавливаемым кешем чатов. У каждой API-реплики нет постоянного каталога оригиналов. Во время синхронного upload тело ограниченно копируется в файл OS temp, файл имеет случайное имя, удаляется в `defer` и нужен только для двукратной проверки/потокового S3 PUT.
Из lab3 сохранены два API-инстанса, HAProxy, PostgreSQL, Redis fallback, bearer-сессии, idempotency сообщений, курсоры, graceful shutdown и persistence/failover hooks. В lab4 добавлены:
- `Attachment` со статусами `pending/ready/failed`, SHA-256, истинным MIME, размером и размерами изображения;
- таблицы `attachments` и `message_attachments`, same-chat foreign keys и deferred constraint «текст или вложение»;
- `attachment_ids` во всех слоях Message без нарушения старого JSON (`[]` по умолчанию);
- binary `POST`, metadata `GET` и авторизованный streaming `GET /content` без redirect;
- общий MinIO volume, idempotent bootstrap приватного bucket и отдельный API user с минимальной policy;
- readiness, зависящий от PostgreSQL и MinIO;
- one-shot reconciler с dry-run по умолчанию, `--apply`, grace period и условными DB updates.
Upload возвращает `201 ready` только после последовательности: полное ограниченное чтение → проверка MIME/PNG/JPEG/пикселей и SHA-256 → insert `pending` → успешный S3 PUT → update `ready`. Ошибка PUT удаляет возможный объект и фиксирует `failed`. Ошибка ответа на финальный DB update считается неоднозначной: update мог уже закоммититься, поэтому объект не удаляется. API делает best-effort readback и либо восстанавливает успешный `201`, либо возвращает `503`, сохранив metadata+object для reconciler. Это исключает состояние `ready` без оригинала при потере ответа PostgreSQL.
Лимиты применяются до неограниченной работы: body читается через `LimitReader(10 MiB + 1)`, `DecodeConfig` проверяет максимум 40 млн пикселей до полного `Decode`, а полный `Decode` проверяет поток/CRC/IDAT/SOS до S3 PUT. Filename не участвует в object key и проверяется на valid UTF-8/blank/path/C0+C1 control characters. Object key формируется сервером как `attachments/{chat_id}/{attachment_id}/original`. На реплику допускаются не более четырёх upload pipeline и одного полного decode одновременно; `GOMEMLIMIT=220MiB` ниже container limit 256 MiB. S3 operation timeout — 5 s, HTTP read/write timeout — 15 s, HAProxy server/client timeout — 30 s, DB query timeout задаётся конфигурацией. Компенсация сначала получает отдельный 1 s budget для DB transition и лишь затем отдельный 1 s budget для best-effort S3 delete.
Публичный `/health/ready` проверяет PostgreSQL и MinIO и возвращает контрактный JSON `503`, если зависимость недоступна. HAProxy использует отдельный внутренний `/health/lb`, отражающий жизнеспособность/drain процесса, поэтому при общем отказе MinIO запросы не заменяются generic HTML-ошибкой балансировщика, а доходят до API и получают согласованный `503 unavailable`.
Соответствие ключевых переменных: `PUBLIC_PORT` — host port HAProxy; `APP_INTERNAL_PORT` — порт API в сети Compose; `DATABASE_URL_FILE`, `CURSOR_SIGNING_KEY_FILE`, `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` — Docker secrets; `S3_ENDPOINT=http://minio:9000`; `S3_BUCKET=messenger-attachments`; `S3_OPERATION_TIMEOUT=5s`.
## Механизм согласованности и трек 5
Гипотеза: периодический inventory PostgreSQL и MinIO с консервативным grace period способен повторяемо исправлять безопасные orphan/missing состояния, не удаляя свежие объекты и вложения, уже связанные с сообщениями.
Reconciler сравнивает все `attachments/` objects с метаданными:
- старый object без metadata удаляется только в apply;
- старый `pending` очищает object и условно становится `failed`;
- старый `ready` без object условно становится `failed`, только если не используется сообщением;
- object у старого `failed` удаляется, только если запись не используется;
- свежие записи/objects игнорируются, а referenced missing помечается `manual_repair_required` без мутации;
- SQL update берёт row lock, затем отдельным statement на свежем READ COMMITTED snapshot повторно проверяет status, cutoff и отсутствие ссылки; concurrent message/reconciler race либо создаёт ссылку, либо переводит attachment в failed, но не оба состояния одновременно;
- object stale-pending удаляется только после успешного условного DB transition, поэтому проигранная гонка приводит к `skipped_concurrent_change`, а не к потере готового оригинала.
Команды оператора:
```sh
make reconcile-dry-run RECONCILE_GRACE=15m
# Просмотреть JSON; затем при согласии:
make reconcile-apply RECONCILE_GRACE=15m
```
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload/message/download/checksum/outsider 404 | [evidence/lab4-results.md](evidence/lab4-results.md) |
| Границы/полная декодировка | `make test-attachment-boundaries` | Ровно 10 MiB и 40 млн пикселей приняты; truncated/corrupt-after-64-KiB отклонены 415; whitespace/C1 filename — 400 | [tests/attachment_boundaries.py](tests/attachment_boundaries.py) |
| Failover API | `make test-failover` с указанными в evidence env | 142 успешных записи, 0 ошибок, recovery 2.771 s; файл читается другой репликой | [evidence/lab4-results.md](evidence/lab4-results.md) |
| Restart и volumes | `make test-persistence` | Сессия, метаданные и bytes пережили restart PostgreSQL/Redis/MinIO/API/frontend/HAProxy | [scripts/restart.sh](scripts/restart.sh) |
| Файловые негативные тесты | `go test ./...` | oversized bytes/pixels, invalid IDAT, truncated PNG/JPEG, corruption after 64 KiB, MIME substitution, unsafe filename, interrupted body, foreign attachment/chat | [internal/application/attachments_test.go](internal/application/attachments_test.go) |
| Неоднозначный DB COMMIT | `go test ./internal/application -count=1` | Потеря ответа после commit восстанавливается readback; при одновременном read outage ready metadata сохраняет оригинал | [internal/application/attachments_test.go](internal/application/attachments_test.go) |
| Реальный отказ MinIO | `make test-storage-outage` | readiness 503 за 2.004 s, upload 503 за 6.016 s; после recovery прежний original неизменен | [tests/storage_outage.py](tests/storage_outage.py) |
| RAM при concurrency ≥10 | `make test-attachment-memory` | 12 × 6,293,028 B, 12/12 success; пики 31,216,107 и 26,214,400 B | [tests/attachment_memory.py](tests/attachment_memory.py) |
| Private bucket / API policy | `make test-minio-policy` | Root secrets/env отсутствуют в API; limited user не создаёт bucket; anonymous GET запрещён | [tests/minio_policy.sh](tests/minio_policy.sh) |
| Reconciler dry-run/apply/repeat | `make test-reconcile` | 4 issues; 2 objects deleted; 2 metadata failed; repeat 0 mutations; referenced защищён | [tests/reconcile_integration.py](tests/reconcile_integration.py) |
| Стоимость обхода | `make test-reconcile-scale` | 100 objects: 9 ms; 1,000 objects: 38 ms core duration | [tests/reconcile_scale.py](tests/reconcile_scale.py) |
| Static/race gates | `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS | [evidence/lab4-results.md](evidence/lab4-results.md) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Memory experiment: один прогон, 12 concurrent uploads, каждый PNG 2048×1024 и 6,293,028 bytes; суммарно 75,516,336 bytes. HAProxy распределил запросы по двум репликам. Docker stats опрашивался во время прогона. 12/12 завершились за 0.599 s, наблюдаемые пики — 29.8 и 25.0 MiB, существенно ниже container limit 256 MiB. После измерения отдельный chat prefix и синтетические DB rows удалены.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
Reconciler integration: в отдельном namespace созданы old orphan object, old pending metadata+object, unreferenced ready metadata без object и referenced ready metadata без object. Dry-run нашёл четыре состояния и не мутировал данные. Apply удалил два objects и перевёл две записи в `failed`. Повторный apply сделал ноль мутаций; referenced missing остался ручной задачей. Синтетические данные удалены в `finally`.
Traversal experiment использовал один и тот же стенд после прогрева, один прогон на размер, отдельный prefix и однобайтовые objects. 100 injected orphans: 109 objects total, 9 ms core и 1,978 ms с запуском one-shot container. 1,000: 1,009 total, 38 ms core и 2,019 ms end-to-end. Это подтверждает повторяемость методики и ожидаемую близкую к линейной стоимость лишь на малом локальном масштабе.
| Воздействие | Ожидание | Наблюдение | Потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Stop одной API-реплики | Вторая продолжает запись/чтение общего файла | recovery 2.771 s, 142 writes, 0 errors | Нет |
| Restart общего стенда | Named volumes сохраняют DB и S3 | persistence PASS | Нет |
| S3 PUT error | 503, metadata failed, object отсутствует | Unit injection PASS | Нет ready-данных |
| MinIO недоступен | readiness/upload дают JSON 503 до API/LB timeout | 2.004 s / 6.016 s, реальный stop/recovery PASS | Старый original неизменен |
| DB finalize response потерян после COMMIT | Readback возвращает ready; при DB outage object не удаляется | Два unit fault-теста PASS | Ready не остаётся без original |
| Orphan/missing/stale состояния | Dry-run, безопасный apply, repeat | Integration PASS | Referenced missing не изменён |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Inventory materializes metadata и object list в RAM; для большого bucket нужны pagination/checkpoints и пакетная выдача issues.
- Один reconciler предполагается запускать оператором; перед scheduler нужны PostgreSQL advisory lock/lease и метрики длительности/ошибок.
- `referenced_missing_object` намеренно не чинится автоматически: требуется восстановление bytes из backup или операторское решение.
- Один-node MinIO и локальные измерения не доказывают HA самого object storage; это учебный стенд.
- Experiment содержит по одному измерению на размер без статистических повторов; для production нужны warm-up, серии, p50/p95/p99, CPU/IO и существенно больший inventory.
- Оригинал проверяется SHA-256 при upload и клиентом в smoke; каждый download не пересчитывает hash на лету.
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"time"
"mtusi2026lab4/internal/adapter/objectstore"
"mtusi2026lab4/internal/adapter/postgres"
"mtusi2026lab4/internal/config"
"mtusi2026lab4/internal/database"
"mtusi2026lab4/internal/reconcile"
)
func main() {
apply := flag.Bool("apply", false, "apply safe repairs; default is dry-run")
gracePeriod := flag.Duration("grace-period", 15*time.Minute,
"ignore metadata and objects newer than this duration")
timeout := flag.Duration("timeout", 2*time.Minute, "maximum reconciliation runtime")
flag.Parse()
if *gracePeriod <= 0 || *timeout <= 0 {
log.Fatal("[FATAL] grace-period and timeout must be positive")
}
cfg, err := config.Load()
if err != nil {
log.Fatalf("[FATAL] Invalid configuration: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
databasePool, err := database.Open(ctx, cfg)
if err != nil {
log.Fatalf("[FATAL] PostgreSQL initialization failed: %v", err)
}
defer databasePool.Close()
metadataStore := postgres.NewPostgreSQLStore(databasePool, cfg.DatabaseQueryTimeout)
objectStore, err := objectstore.NewMinIOStore(
cfg.S3Endpoint,
cfg.S3Region,
cfg.S3Bucket,
cfg.S3AccessKeyID,
cfg.S3SecretAccessKey,
cfg.S3OperationTimeout,
)
if err != nil {
log.Fatalf("[FATAL] Invalid object storage configuration: %v", err)
}
if err = objectStore.Ping(ctx); err != nil {
log.Fatalf("[FATAL] Object storage unavailable: %v", err)
}
reconciler, err := reconcile.New(metadataStore, objectStore)
if err != nil {
log.Fatalf("[FATAL] Reconciler initialization failed: %v", err)
}
result, err := reconciler.Run(ctx, *gracePeriod, *apply)
if err != nil {
log.Fatalf("[FATAL] Reconciliation failed: %v", err)
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
if err = encoder.Encode(result); err != nil {
log.Fatalf("[FATAL] Encode reconciliation result: %v", err)
}
if !*apply && len(result.Issues) > 0 {
fmt.Fprintln(os.Stderr, "dry-run found reconciliation issues; rerun with --apply after review")
}
}
+36 -1
View File
@@ -11,6 +11,8 @@ import (
"time"
cacheadapter "mtusi2026lab4/internal/adapter/cache"
"mtusi2026lab4/internal/adapter/health"
"mtusi2026lab4/internal/adapter/objectstore"
"mtusi2026lab4/internal/adapter/postgres"
redisadapter "mtusi2026lab4/internal/adapter/redis"
"mtusi2026lab4/internal/application"
@@ -43,6 +45,25 @@ func main() {
databasePool,
cfg.DatabaseQueryTimeout,
)
minioStore, err := objectstore.NewMinIOStore(
cfg.S3Endpoint,
cfg.S3Region,
cfg.S3Bucket,
cfg.S3AccessKeyID,
cfg.S3SecretAccessKey,
cfg.S3OperationTimeout,
)
if err != nil {
log.Fatalf("[FATAL] Invalid object storage configuration: %v", err)
}
storageContext, storageCancel := context.WithTimeout(
context.Background(),
cfg.S3OperationTimeout,
)
if err = minioStore.Ping(storageContext); err != nil {
log.Printf("[WARN] Object storage unavailable at startup: %v", err)
}
storageCancel()
var applicationStore application.Repository = postgresStore
@@ -90,6 +111,7 @@ func main() {
cursorCodec,
cfg.SessionTTL,
application.WithIdempotencyTTL(cfg.IdempotencyTTL),
application.WithObjectStore(minioStore),
)
if err != nil {
log.Fatalf("[FATAL] Application initialization failed: %v", err)
@@ -98,7 +120,7 @@ func main() {
handler := handlers.NewHandler(
service,
handlers.WithReadiness(
databasePool,
health.NewMultiChecker(databasePool, minioStore),
cfg.ReadinessTimeout,
),
handlers.WithSessionCookieSecure(cfg.SessionCookieSecure),
@@ -107,6 +129,7 @@ func main() {
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", handler.HealthLive)
mux.HandleFunc("GET /health/lb", handler.HealthLive)
mux.HandleFunc("GET /health/ready", handler.HealthReady)
mux.HandleFunc(
@@ -150,6 +173,18 @@ func main() {
"GET /api/v1/chats/{chat_id}/messages",
requireAuth(handler.GetMessages),
)
mux.HandleFunc(
"POST /api/v1/chats/{chat_id}/attachments",
requireAuth(handler.UploadAttachment),
)
mux.HandleFunc(
"GET /api/v1/attachments/{attachment_id}",
requireAuth(handler.GetAttachment),
)
mux.HandleFunc(
"GET /api/v1/attachments/{attachment_id}/content",
requireAuth(handler.DownloadAttachment),
)
var rootHandler http.Handler = mux
rootHandler = middleware.Recovery(rootHandler)
+94 -2
View File
@@ -5,7 +5,7 @@ services:
dockerfile: infrastructure/haproxy/Dockerfile
image: mtusi2026lab4-load-balancer
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
- "127.0.0.1:${PUBLIC_PORT:-8080}:8080"
depends_on:
api:
condition: service_healthy
@@ -128,13 +128,84 @@ services:
cpus: "0.25"
memory: 64M
minio:
build:
context: .
dockerfile: infrastructure/minio/Dockerfile
target: server
args:
MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z
image: mtusi2026lab4-minio:RELEASE.2025-10-15T17-29-55Z
command: ["server", "/data", "--address", ":9000"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
MINIO_BROWSER: "off"
secrets:
- minio_root_user
- minio_root_password
volumes:
- minio_data:/data
expose:
- "9000"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.75"
memory: 512M
minio-bootstrap:
build:
context: .
dockerfile: infrastructure/minio/Dockerfile
target: client
args:
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
image: mtusi2026lab4-mc:RELEASE.2025-08-13T08-35-41Z
entrypoint: ["/bin/sh", "-ec"]
command:
- |
root_user="$$(cat /run/secrets/minio_root_user)"
root_password="$$(cat /run/secrets/minio_root_password)"
api_user="$$(cat /run/secrets/s3_access_key_id)"
api_password="$$(cat /run/secrets/s3_secret_access_key)"
mc alias set local http://minio:9000 "$$root_user" "$$root_password" --api S3v4
mc mb --ignore-existing "local/$${S3_BUCKET}"
mc anonymous set none "local/$${S3_BUCKET}"
sed "s/__BUCKET__/$${S3_BUCKET}/g" /config/api-policy.template.json > /tmp/api-policy.json
mc admin policy create local messenger-api /tmp/api-policy.json
if ! mc admin user info local "$$api_user" >/dev/null 2>&1; then
mc admin user add local "$$api_user" "$$api_password"
fi
mc admin policy attach local messenger-api --user "$$api_user"
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infrastructure/minio/api-policy.template.json:/config/api-policy.template.json:ro
depends_on:
minio:
condition: service_healthy
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_PORT: ${APP_INTERNAL_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
@@ -148,6 +219,13 @@ services:
CACHE_OPERATION_TIMEOUT: ${CACHE_OPERATION_TIMEOUT:-250ms}
CACHE_POOL_SIZE: ${CACHE_POOL_SIZE:-10}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
S3_OPERATION_TIMEOUT: ${S3_OPERATION_TIMEOUT:-5s}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
@@ -161,8 +239,11 @@ services:
READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s}
SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s}
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s}
GOMEMLIMIT: ${API_GO_MEMORY_LIMIT:-220MiB}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
- cursor_signing_key
expose:
- "8080"
@@ -173,6 +254,8 @@ services:
condition: service_healthy
migrate:
condition: service_completed_successfully
minio-bootstrap:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
resources:
@@ -240,6 +323,7 @@ services:
volumes:
postgres_data:
redis_insight_data:
minio_data:
postgres_restore_data:
secrets:
@@ -249,3 +333,11 @@ secrets:
file: ./secrets/database_url.txt
cursor_signing_key:
file: ./secrets/cursor_signing_key.txt
minio_root_user:
file: ./secrets/minio_root_user.txt
minio_root_password:
file: ./secrets/minio_root_password.txt
s3_access_key_id:
file: ./secrets/s3_access_key_id.txt
s3_secret_access_key:
file: ./secrets/s3_secret_access_key.txt
+19
View File
@@ -0,0 +1,19 @@
# Lab 4 verification summary
Date: 2026-10-29. Environment: local Docker Compose on WSL2; two API replicas; PostgreSQL 16.15; Redis 8.10.2; MinIO release `RELEASE.2025-10-15T17-29-55Z` built from the official source module.
| Check | Command | Observed result |
| --- | --- | --- |
| Public contract | `BASE_URL=http://127.0.0.1:18080 make test` | PASS: upload, attachment-only message, authorized byte download, checksum/content type and outsider 404 |
| Exact boundaries / decode | `BASE_URL=http://127.0.0.1:18080 make test-attachment-boundaries` | Exact 10 MiB and 40,000,000 pixels accepted; truncated/corrupted-after-64-KiB PNG rejected 415; blank/C1 filename rejected 400 |
| One-replica failover | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/stop-one.sh make test-failover` | 2 instances before failure; 142 successful writes; 0 failed attempts; first recovery 2.771 s; attachment unchanged |
| Full persistence restart | `PUBLIC_PORT=18080 BASE_URL=http://127.0.0.1:18080 ACTION_SCRIPT=./scripts/restart.sh make test-persistence` | PASS after restarting PostgreSQL, Redis, MinIO, both API replicas, frontend and HAProxy; existing session, metadata and bytes survived |
| Real MinIO outage | `BASE_URL=http://127.0.0.1:18080 make test-storage-outage` | Public readiness returned JSON 503 in 2.004 s, upload returned JSON 503 in 6.016 s; after recovery the 111,817-byte previous original was byte-identical |
| Private bucket / least privilege | `make test-minio-policy` | API containers had no MinIO root env/secrets; limited credentials could put/read/delete only in configured bucket; creating another bucket and anonymous GET failed |
| Concurrent upload memory | `BASE_URL=http://127.0.0.1:18080 make test-attachment-memory` | 12/12 successful 6,293,028-byte PNG uploads (75,516,336 bytes total) in 0.599 s; peak API memory 31,216,107 and 26,214,400 bytes |
| Reconciler behavior | `make test-reconcile` | Dry-run found 4 injected issues; apply deleted 2 objects and marked 2 attachments failed; repeat made 0 mutations; 1 referenced missing attachment remained protected/manual |
| Reconciler traversal | `make test-reconcile-scale` | 100 orphan objects: 109 objects scanned, 9 ms core / 1,978 ms including container startup. 1,000 orphans: 1,009 scanned, 38 ms core / 2,019 ms including startup |
| Ambiguous DB COMMIT fault | `go test ./internal/application -count=1` | Successful finalize with lost response is recovered by readback; if DB remains unavailable, ready metadata keeps its object and returns an honest error |
| Go quality gates | `go test ./...`; `go test -race ./...`; `go vet ./...`; `make check`; `git diff --check` | PASS |
The measurements are single local runs, not production capacity claims. The reconciler currently materializes both inventories in memory, so its measured near-linear small-scale cost must be re-evaluated before using it with a large bucket.
+7 -2
View File
@@ -143,7 +143,12 @@ export const api = {
);
},
sendMessage(chatID: string, text: string, idempotencyKey: string): Promise<Message> {
sendMessage(
chatID: string,
text: string,
idempotencyKey: string,
attachmentIDs: string[] = [],
): Promise<Message> {
return request<Message>(
`${browserApi}/chats/${encodeURIComponent(chatID)}/messages`,
{
@@ -151,7 +156,7 @@ export const api = {
headers: {
"Idempotency-Key": idempotencyKey,
},
body: { text },
body: { text, attachment_ids: attachmentIDs },
},
);
},
+1
View File
@@ -24,6 +24,7 @@ export type Message = {
chat_id: string;
sender_id: string;
text: string;
attachment_ids: string[];
created_at: string;
};
+14
View File
@@ -5,17 +5,31 @@ go 1.27.1
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.4.1
github.com/redis/go-redis/v9 v9.23.0
golang.org/x/crypto v0.57.0
)
require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.uber.org/atomic v1.12.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
+34
View File
@@ -5,6 +5,9 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@@ -15,16 +18,42 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
@@ -33,6 +62,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
@@ -40,4 +71,7 @@ golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+2 -2
View File
@@ -43,7 +43,7 @@ frontend public
backend api_backend
balance leastconn
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 8 api:8080
@@ -55,7 +55,7 @@ backend message_backend
option http-buffer-request
retries 3
retry-on conn-failure empty-response response-timeout 502 503 504
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check send meth GET uri /health/lb ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 8 api:8080
+40
View File
@@ -0,0 +1,40 @@
FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS minio-builder
ARG MINIO_VERSION=RELEASE.2025-10-15T17-29-55Z
RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/minio@${MINIO_VERSION}
FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS mc-builder
ARG MC_VERSION=RELEASE.2025-08-13T08-35-41Z
RUN CGO_ENABLED=0 GOBIN=/out go install github.com/minio/mc@${MC_VERSION}
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS server
RUN apk add --no-cache ca-certificates wget && \
addgroup -g 1000 minio && \
adduser -u 1000 -G minio -s /sbin/nologin -D minio && \
mkdir -p /data && \
chown minio:minio /data
COPY --from=minio-builder /out/minio /usr/local/bin/minio
USER minio:minio
VOLUME ["/data"]
EXPOSE 9000
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=20 \
CMD wget -qO- http://127.0.0.1:9000/minio/health/live >/dev/null || exit 1
ENTRYPOINT ["/usr/local/bin/minio"]
CMD ["server", "/data", "--address", ":9000"]
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 AS client
RUN apk add --no-cache ca-certificates && \
addgroup -g 1000 minio && \
adduser -u 1000 -G minio -s /sbin/nologin -D minio
COPY --from=mc-builder /out/mc /usr/local/bin/mc
USER minio:minio
ENTRYPOINT ["/usr/local/bin/mc"]
@@ -0,0 +1,26 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::__BUCKET__"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::__BUCKET__/*"
]
}
]
}
+33
View File
@@ -0,0 +1,33 @@
package health
import (
"context"
"fmt"
)
type Checker interface {
Ping(ctx context.Context) error
}
type MultiChecker struct {
checkers []Checker
}
func NewMultiChecker(checkers ...Checker) *MultiChecker {
filtered := make([]Checker, 0, len(checkers))
for _, checker := range checkers {
if checker != nil {
filtered = append(filtered, checker)
}
}
return &MultiChecker{checkers: filtered}
}
func (checker *MultiChecker) Ping(ctx context.Context) error {
for index, dependency := range checker.checkers {
if err := dependency.Ping(ctx); err != nil {
return fmt.Errorf("dependency %d: %w", index, err)
}
}
return nil
}
+83 -1
View File
@@ -166,6 +166,15 @@ func (r *MemoryRepository) CreateMessageIdempotently(
if _, err := r.legacy.GetChatByID(write.ChatID, write.ActorID); err != nil {
return domain.Message{}, err
}
for _, attachmentID := range write.AttachmentIDs {
attachment, err := r.legacy.GetAttachmentByID(attachmentID)
if err != nil || attachment.ChatID != write.ChatID {
return domain.Message{}, ErrNotFound
}
if attachment.Status != domain.AttachmentStatusReady {
return domain.Message{}, application.ErrAttachmentNotReady
}
}
if stored, exists := r.idempotencyByScope[scope]; exists {
if stored.expiresAt.After(now) {
@@ -177,7 +186,12 @@ func (r *MemoryRepository) CreateMessageIdempotently(
delete(r.idempotencyByScope, scope)
}
message, err := r.legacy.CreateMessage(write.ChatID, write.ActorID, write.Text)
message, err := r.legacy.CreateMessage(
write.ChatID,
write.ActorID,
write.Text,
write.AttachmentIDs,
)
if err != nil {
return domain.Message{}, err
}
@@ -277,6 +291,74 @@ func (r *MemoryRepository) DeleteSession(
return nil
}
func (r *MemoryRepository) CreateAttachment(
ctx context.Context,
attachment domain.Attachment,
) (domain.Attachment, error) {
if err := ctx.Err(); err != nil {
return domain.Attachment{}, err
}
if err := r.legacy.PutAttachment(attachment); err != nil {
return domain.Attachment{}, err
}
return attachment, nil
}
func (r *MemoryRepository) MarkAttachmentReady(
ctx context.Context,
attachmentID string,
updatedAt time.Time,
) (domain.Attachment, error) {
if err := ctx.Err(); err != nil {
return domain.Attachment{}, err
}
return r.legacy.UpdateAttachmentStatus(
attachmentID,
domain.AttachmentStatusReady,
nil,
updatedAt,
)
}
func (r *MemoryRepository) MarkAttachmentFailed(
ctx context.Context,
attachmentID string,
errorCode string,
updatedAt time.Time,
) error {
if err := ctx.Err(); err != nil {
return err
}
_, err := r.legacy.UpdateAttachmentStatus(
attachmentID,
domain.AttachmentStatusFailed,
&errorCode,
updatedAt,
)
return err
}
func (r *MemoryRepository) GetAttachment(
ctx context.Context,
attachmentID string,
callerID string,
) (domain.Attachment, error) {
if err := ctx.Err(); err != nil {
return domain.Attachment{}, err
}
attachment, err := r.legacy.GetAttachmentByID(attachmentID)
if err != nil {
return domain.Attachment{}, err
}
if _, err = r.legacy.GetChatByID(attachment.ChatID, callerID); err != nil {
return domain.Attachment{}, err
}
if attachment.Variants == nil {
attachment.Variants = make([]domain.AttachmentVariant, 0)
}
return attachment, nil
}
func memoryTokenHashKey(tokenHash []byte) ([32]byte, error) {
var key [32]byte
+90 -6
View File
@@ -20,6 +20,8 @@ type MemoryStore struct {
chatsByID map[string]domain.Chat //key = chat UUID
messagesByChatID map[string][]domain.Message // key = chat UUID, value = слайс сообщений
attachmentsByID map[string]domain.Attachment
}
func NewMemoryStore() *MemoryStore {
@@ -28,6 +30,7 @@ func NewMemoryStore() *MemoryStore {
usersByUsername: make(map[string]string),
chatsByID: make(map[string]domain.Chat),
messagesByChatID: make(map[string][]domain.Message),
attachmentsByID: make(map[string]domain.Attachment),
}
}
@@ -177,12 +180,86 @@ func (s *MemoryStore) GetChatsForUser(
return items, end < len(chats), nil
}
// PutAttachment stores metadata for adapter and application tests. Binary
// content intentionally does not belong to this in-memory metadata store.
func (s *MemoryStore) PutAttachment(attachment domain.Attachment) error {
s.mu.Lock()
defer s.mu.Unlock()
if _, exists := s.attachmentsByID[attachment.ID]; exists {
return ErrConflict
}
chat, exists := s.chatsByID[attachment.ChatID]
if !exists {
return ErrNotFound
}
ownerIsMember := false
for _, memberID := range chat.MemberIDs {
if memberID == attachment.OwnerID {
ownerIsMember = true
break
}
}
if !ownerIsMember {
return ErrNotFound
}
s.attachmentsByID[attachment.ID] = attachment
return nil
}
func (s *MemoryStore) GetAttachmentByID(
attachmentID string,
) (domain.Attachment, error) {
s.mu.RLock()
defer s.mu.RUnlock()
attachment, exists := s.attachmentsByID[attachmentID]
if !exists {
return domain.Attachment{}, ErrNotFound
}
return attachment, nil
}
func (s *MemoryStore) UpdateAttachmentStatus(
attachmentID string,
status domain.AttachmentStatus,
errorCode *string,
updatedAt time.Time,
) (domain.Attachment, error) {
s.mu.Lock()
defer s.mu.Unlock()
attachment, exists := s.attachmentsByID[attachmentID]
if !exists {
return domain.Attachment{}, ErrNotFound
}
if attachment.Status != domain.AttachmentStatusPending {
return domain.Attachment{}, ErrConflict
}
attachment.Status = status
attachment.ErrorCode = errorCode
attachment.UpdatedAt = updatedAt
if attachment.Variants == nil {
attachment.Variants = make([]domain.AttachmentVariant, 0)
}
s.attachmentsByID[attachmentID] = attachment
return attachment, nil
}
// ---Сообщения---
func (s *MemoryStore) CreateMessage(
chatID, senderID, text string,
attachmentLists ...[]string,
) (domain.Message, error) {
messageID := uuid.NewString()
attachmentIDs := make([]string, 0)
if len(attachmentLists) > 0 {
attachmentIDs = append(attachmentIDs, attachmentLists[0]...)
}
s.mu.Lock()
defer s.mu.Unlock()
@@ -216,11 +293,12 @@ func (s *MemoryStore) CreateMessage(
}
message := domain.Message{
ID: messageID,
ChatID: chatID,
SenderID: senderID,
Text: text,
CreatedAt: createdAt,
ID: messageID,
ChatID: chatID,
SenderID: senderID,
Text: text,
AttachmentIDs: attachmentIDs,
CreatedAt: createdAt,
}
s.messagesByChatID[chatID] = append(messages, message)
@@ -282,6 +360,12 @@ func (s *MemoryStore) GetMessages(
// копируем только возвращаемую страницу, а не всю историю, изменяя внутренний срез
items := make([]domain.Message, end-start)
copy(items, messages[start:end])
for index, message := range messages[start:end] {
message.AttachmentIDs = append(
make([]string, 0, len(message.AttachmentIDs)),
message.AttachmentIDs...,
)
items[index] = message
}
return items, end < len(messages), nil
}
+201
View File
@@ -0,0 +1,201 @@
package objectstore
import (
"bytes"
"context"
"fmt"
"io"
"net/url"
"strings"
"time"
"mtusi2026lab4/internal/application"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type MinIOStore struct {
client *minio.Client
bucket string
operationTimeout time.Duration
}
type StoredObject struct {
Key string
SizeBytes int64
LastModified time.Time
}
func NewMinIOStore(
endpoint string,
region string,
bucket string,
accessKeyID string,
secretAccessKey string,
operationTimeout time.Duration,
) (*MinIOStore, error) {
parsedEndpoint, err := url.Parse(endpoint)
if err != nil || parsedEndpoint.Host == "" {
return nil, fmt.Errorf("S3 endpoint must be an absolute HTTP(S) URL")
}
if parsedEndpoint.Scheme != "http" && parsedEndpoint.Scheme != "https" {
return nil, fmt.Errorf("S3 endpoint scheme must be http or https")
}
if parsedEndpoint.Path != "" && parsedEndpoint.Path != "/" {
return nil, fmt.Errorf("S3 endpoint must not contain a path")
}
if parsedEndpoint.RawQuery != "" || parsedEndpoint.Fragment != "" || parsedEndpoint.User != nil {
return nil, fmt.Errorf("S3 endpoint must not contain credentials, query or fragment")
}
if strings.TrimSpace(bucket) == "" || strings.TrimSpace(accessKeyID) == "" ||
strings.TrimSpace(secretAccessKey) == "" {
return nil, fmt.Errorf("S3 bucket and credentials are required")
}
if operationTimeout <= 0 {
return nil, fmt.Errorf("S3 operation timeout must be positive")
}
client, err := minio.New(parsedEndpoint.Host, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: parsedEndpoint.Scheme == "https",
Region: region,
})
if err != nil {
return nil, fmt.Errorf("create MinIO client: %w", err)
}
return &MinIOStore{client: client, bucket: bucket, operationTimeout: operationTimeout}, nil
}
func (store *MinIOStore) Put(ctx context.Context, object application.ObjectPut) error {
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
defer cancel()
upload, err := store.client.PutObject(
operationContext,
store.bucket,
object.Key,
object.Body,
object.SizeBytes,
minio.PutObjectOptions{ContentType: object.ContentType},
)
if err != nil {
return fmt.Errorf("put MinIO object: %w", err)
}
if upload.Size != object.SizeBytes {
return fmt.Errorf("put MinIO object: stored %d bytes, want %d", upload.Size, object.SizeBytes)
}
return nil
}
func (store *MinIOStore) Get(
ctx context.Context,
key string,
) (application.ObjectContent, error) {
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
stat, err := store.client.StatObject(
operationContext,
store.bucket,
key,
minio.StatObjectOptions{},
)
if err != nil {
cancel()
return application.ObjectContent{}, fmt.Errorf("stat MinIO object: %w", err)
}
object, err := store.client.GetObject(
operationContext,
store.bucket,
key,
minio.GetObjectOptions{},
)
if err != nil {
cancel()
return application.ObjectContent{}, fmt.Errorf("get MinIO object: %w", err)
}
firstByte := make([]byte, 1)
readCount, readError := object.Read(firstByte)
if readError != nil && readCount == 0 {
_ = object.Close()
cancel()
return application.ObjectContent{}, fmt.Errorf("read MinIO object: %w", readError)
}
if readCount != 1 {
_ = object.Close()
cancel()
return application.ObjectContent{}, fmt.Errorf("read MinIO object: expected non-empty object")
}
return application.ObjectContent{
Body: &cancelingReadCloser{
Reader: io.MultiReader(bytes.NewReader(firstByte), object),
Closer: object,
cancel: cancel,
},
SizeBytes: stat.Size,
}, nil
}
func (store *MinIOStore) List(
ctx context.Context,
prefix string,
) ([]StoredObject, error) {
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
defer cancel()
objects := make([]StoredObject, 0)
for object := range store.client.ListObjects(
operationContext,
store.bucket,
minio.ListObjectsOptions{Prefix: prefix, Recursive: true},
) {
if object.Err != nil {
return nil, fmt.Errorf("list MinIO objects: %w", object.Err)
}
objects = append(objects, StoredObject{
Key: object.Key,
SizeBytes: object.Size,
LastModified: object.LastModified.UTC(),
})
}
return objects, nil
}
func (store *MinIOStore) Delete(ctx context.Context, key string) error {
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
defer cancel()
if err := store.client.RemoveObject(
operationContext,
store.bucket,
key,
minio.RemoveObjectOptions{},
); err != nil {
return fmt.Errorf("delete MinIO object: %w", err)
}
return nil
}
func (store *MinIOStore) Ping(ctx context.Context) error {
operationContext, cancel := context.WithTimeout(ctx, store.operationTimeout)
defer cancel()
exists, err := store.client.BucketExists(operationContext, store.bucket)
if err != nil {
return fmt.Errorf("check MinIO bucket: %w", err)
}
if !exists {
return fmt.Errorf("MinIO bucket %q does not exist", store.bucket)
}
return nil
}
type cancelingReadCloser struct {
io.Reader
io.Closer
cancel context.CancelFunc
}
func (reader *cancelingReadCloser) Close() error {
err := reader.Closer.Close()
reader.cancel()
return err
}
var _ application.ObjectStore = (*MinIOStore)(nil)
@@ -0,0 +1,37 @@
package objectstore
import (
"testing"
"time"
)
func TestNewMinIOStoreValidatesConfiguration(t *testing.T) {
testCases := []struct {
name string
endpoint string
bucket string
accessKey string
secretKey string
timeout time.Duration
wantError bool
}{
{name: "valid HTTP", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second},
{name: "valid HTTPS", endpoint: "https://s3.example.test", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second},
{name: "relative endpoint", endpoint: "minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
{name: "endpoint path", endpoint: "http://minio:9000/path", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
{name: "embedded credentials", endpoint: "http://user:pass@minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
{name: "empty bucket", endpoint: "http://minio:9000", accessKey: "api", secretKey: "secret", timeout: time.Second, wantError: true},
{name: "empty credentials", endpoint: "http://minio:9000", bucket: "attachments", timeout: time.Second, wantError: true},
{name: "invalid timeout", endpoint: "http://minio:9000", bucket: "attachments", accessKey: "api", secretKey: "secret", wantError: true},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
_, err := NewMinIOStore(testCase.endpoint, "us-east-1", testCase.bucket,
testCase.accessKey, testCase.secretKey, testCase.timeout)
if (err != nil) != testCase.wantError {
t.Fatalf("NewMinIOStore() error = %v, wantError %v", err, testCase.wantError)
}
})
}
}
+306
View File
@@ -0,0 +1,306 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"mtusi2026lab4/internal/domain"
"mtusi2026lab4/internal/reconcile"
"github.com/jackc/pgx/v5"
)
func (s *PostgreSQLStore) CreateAttachment(
ctx context.Context,
attachment domain.Attachment,
) (domain.Attachment, error) {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
err := s.pool.QueryRow(
queryContext,
`
INSERT INTO attachments (
id, chat_id, owner_id, object_key, filename, content_type,
size_bytes, sha256, width, height, status, error_code,
created_at, updated_at
)
VALUES (
$1, $2, $3, $4, $5, $6,
$7, decode($8, 'hex'), $9, $10, $11, $12,
$13, $14
)
RETURNING created_at, updated_at
`,
attachment.ID,
attachment.ChatID,
attachment.OwnerID,
attachment.ObjectKey,
attachment.Filename,
attachment.ContentType,
attachment.SizeBytes,
attachment.SHA256,
attachment.Width,
attachment.Height,
attachment.Status,
attachment.ErrorCode,
attachment.CreatedAt,
attachment.UpdatedAt,
).Scan(&attachment.CreatedAt, &attachment.UpdatedAt)
if err != nil {
return domain.Attachment{}, fmt.Errorf("insert attachment metadata: %w", err)
}
if attachment.Variants == nil {
attachment.Variants = make([]domain.AttachmentVariant, 0)
}
return attachment, nil
}
func (s *PostgreSQLStore) MarkAttachmentReady(
ctx context.Context,
attachmentID string,
updatedAt time.Time,
) (domain.Attachment, error) {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
var attachment domain.Attachment
var sha256Bytes []byte
err := s.pool.QueryRow(
queryContext,
`
UPDATE attachments
SET status = 'ready', error_code = NULL, updated_at = $2
WHERE id = $1 AND status = 'pending'
RETURNING
id::text, chat_id::text, owner_id::text, object_key,
filename, content_type, size_bytes, sha256, width, height,
status, error_code, created_at, updated_at
`,
attachmentID,
updatedAt,
).Scan(
&attachment.ID,
&attachment.ChatID,
&attachment.OwnerID,
&attachment.ObjectKey,
&attachment.Filename,
&attachment.ContentType,
&attachment.SizeBytes,
&sha256Bytes,
&attachment.Width,
&attachment.Height,
&attachment.Status,
&attachment.ErrorCode,
&attachment.CreatedAt,
&attachment.UpdatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return domain.Attachment{}, ErrConflict
}
if err != nil {
return domain.Attachment{}, fmt.Errorf("mark attachment ready: %w", err)
}
attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes)
attachment.Variants = make([]domain.AttachmentVariant, 0)
return attachment, nil
}
func (s *PostgreSQLStore) MarkAttachmentFailed(
ctx context.Context,
attachmentID string,
errorCode string,
updatedAt time.Time,
) error {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
commandTag, err := s.pool.Exec(
queryContext,
`
UPDATE attachments
SET status = 'failed', error_code = $2, updated_at = $3
WHERE id = $1 AND status = 'pending'
`,
attachmentID,
errorCode,
updatedAt,
)
if err != nil {
return fmt.Errorf("mark attachment failed: %w", err)
}
if commandTag.RowsAffected() != 1 {
return ErrConflict
}
return nil
}
func (s *PostgreSQLStore) GetAttachment(
ctx context.Context,
attachmentID string,
callerID string,
) (domain.Attachment, error) {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
var attachment domain.Attachment
var sha256Bytes []byte
err := s.pool.QueryRow(
queryContext,
`
SELECT
a.id::text, a.chat_id::text, a.owner_id::text, a.object_key,
a.filename, a.content_type, a.size_bytes, a.sha256,
a.width, a.height, a.status, a.error_code,
a.created_at, a.updated_at
FROM attachments AS a
JOIN chat_members AS cm
ON cm.chat_id = a.chat_id AND cm.user_id = $2
WHERE a.id = $1
`,
attachmentID,
callerID,
).Scan(
&attachment.ID,
&attachment.ChatID,
&attachment.OwnerID,
&attachment.ObjectKey,
&attachment.Filename,
&attachment.ContentType,
&attachment.SizeBytes,
&sha256Bytes,
&attachment.Width,
&attachment.Height,
&attachment.Status,
&attachment.ErrorCode,
&attachment.CreatedAt,
&attachment.UpdatedAt,
)
if errors.Is(err, pgx.ErrNoRows) {
return domain.Attachment{}, ErrNotFound
}
if err != nil {
return domain.Attachment{}, fmt.Errorf("get attachment metadata: %w", err)
}
attachment.SHA256 = fmt.Sprintf("%x", sha256Bytes)
attachment.Variants = make([]domain.AttachmentVariant, 0)
return attachment, nil
}
func (s *PostgreSQLStore) ListAttachmentInventory(
ctx context.Context,
) ([]reconcile.AttachmentRecord, error) {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(
queryContext,
`
SELECT
a.id::text,
a.object_key,
a.status,
a.updated_at,
EXISTS (
SELECT 1
FROM message_attachments AS ma
WHERE ma.attachment_id = a.id
) AS referenced
FROM attachments AS a
ORDER BY a.id
`,
)
if err != nil {
return nil, fmt.Errorf("query attachment inventory: %w", err)
}
defer rows.Close()
records := make([]reconcile.AttachmentRecord, 0)
for rows.Next() {
var record reconcile.AttachmentRecord
if err = rows.Scan(
&record.ID,
&record.ObjectKey,
&record.Status,
&record.UpdatedAt,
&record.Referenced,
); err != nil {
return nil, fmt.Errorf("scan attachment inventory: %w", err)
}
records = append(records, record)
}
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("iterate attachment inventory: %w", err)
}
return records, nil
}
func (s *PostgreSQLStore) MarkAttachmentFailedIfUnreferenced(
ctx context.Context,
attachmentID string,
expectedStatus domain.AttachmentStatus,
updatedBefore time.Time,
errorCode string,
updatedAt time.Time,
) (bool, error) {
queryContext, cancel := s.withTimeout(ctx)
defer cancel()
transaction, err := s.pool.BeginTx(queryContext, pgx.TxOptions{})
if err != nil {
return false, fmt.Errorf("begin attachment reconciliation transaction: %w", err)
}
defer func() { _ = transaction.Rollback(queryContext) }()
var lockedAttachmentID string
err = transaction.QueryRow(
queryContext,
`
SELECT id::text
FROM attachments
WHERE id = $1
FOR UPDATE
`,
attachmentID,
).Scan(&lockedAttachmentID)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("lock attachment during reconciliation: %w", err)
}
// This is deliberately a second statement. Under READ COMMITTED it gets a
// fresh snapshot after the row lock was acquired, so a message transaction
// that linked this attachment while we waited is visible to NOT EXISTS.
commandTag, err := transaction.Exec(
queryContext,
`
UPDATE attachments AS a
SET status = 'failed', error_code = $4, updated_at = $5
WHERE a.id = $1
AND a.status = $2
AND a.updated_at <= $3
AND NOT EXISTS (
SELECT 1
FROM message_attachments AS ma
WHERE ma.attachment_id = a.id
)
`,
attachmentID,
expectedStatus,
updatedBefore,
errorCode,
updatedAt,
)
if err != nil {
return false, fmt.Errorf("mark attachment failed during reconciliation: %w", err)
}
if err = transaction.Commit(queryContext); err != nil {
return false, fmt.Errorf("commit attachment reconciliation: %w", err)
}
return commandTag.RowsAffected() == 1, nil
}
var _ reconcile.MetadataStore = (*PostgreSQLStore)(nil)
+109 -7
View File
@@ -48,6 +48,59 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
return domain.Message{}, ErrNotFound
}
attachmentUUIDs := make([]uuid.UUID, len(write.AttachmentIDs))
for index, attachmentID := range write.AttachmentIDs {
parsedID, parseErr := uuid.Parse(attachmentID)
if parseErr != nil {
return domain.Message{}, fmt.Errorf("parse validated attachment id: %w", parseErr)
}
attachmentUUIDs[index] = parsedID
}
if len(attachmentUUIDs) > 0 {
rows, queryError := transaction.Query(
queryContext,
`
SELECT id::text, status
FROM attachments
WHERE chat_id = $1
AND id = ANY($2::uuid[])
ORDER BY id
FOR UPDATE
`,
write.ChatID,
attachmentUUIDs,
)
if queryError != nil {
return domain.Message{}, fmt.Errorf("lock message attachments: %w", queryError)
}
availableCount := 0
readyCount := 0
for rows.Next() {
var attachmentID string
var status domain.AttachmentStatus
if queryError = rows.Scan(&attachmentID, &status); queryError != nil {
rows.Close()
return domain.Message{}, fmt.Errorf("scan locked message attachment: %w", queryError)
}
availableCount++
if status == domain.AttachmentStatusReady {
readyCount++
}
}
queryError = rows.Err()
rows.Close()
if queryError != nil {
return domain.Message{}, fmt.Errorf("iterate locked message attachments: %w", queryError)
}
if availableCount != len(attachmentUUIDs) {
return domain.Message{}, ErrNotFound
}
if readyCount != len(attachmentUUIDs) {
return domain.Message{}, application.ErrAttachmentNotReady
}
}
_, err = transaction.Exec(
queryContext,
`
@@ -68,10 +121,11 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
}
message := domain.Message{
ID: uuid.NewString(),
ChatID: write.ChatID,
SenderID: write.ActorID,
Text: write.Text,
ID: uuid.NewString(),
ChatID: write.ChatID,
SenderID: write.ActorID,
Text: write.Text,
AttachmentIDs: append([]string(nil), write.AttachmentIDs...),
}
var storedHash []byte
@@ -141,7 +195,15 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
chat_id::text,
sender_id::text,
text,
created_at
created_at,
COALESCE(
ARRAY(
SELECT attachment_id::text
FROM message_attachments
WHERE message_id = messages.id
ORDER BY position
), ARRAY[]::text[]
)
FROM messages
WHERE id = $1
`,
@@ -152,6 +214,7 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
)
if err != nil {
return domain.Message{}, fmt.Errorf("read idempotent message: %w", err)
@@ -187,6 +250,28 @@ func (s *PostgreSQLStore) CreateMessageIdempotently(
return domain.Message{}, fmt.Errorf("insert message: %w", err)
}
if len(attachmentUUIDs) > 0 {
_, err = transaction.Exec(
queryContext,
`
INSERT INTO message_attachments (
message_id,
chat_id,
attachment_id,
position
)
SELECT $1, $2, attachment_id, (ordinality - 1)::smallint
FROM unnest($3::uuid[]) WITH ORDINALITY AS input(attachment_id, ordinality)
`,
message.ID,
message.ChatID,
attachmentUUIDs,
)
if err != nil {
return domain.Message{}, fmt.Errorf("link message attachments: %w", err)
}
}
if err = transaction.Commit(queryContext); err != nil {
return domain.Message{}, fmt.Errorf("commit idempotent message: %w", err)
}
@@ -241,7 +326,15 @@ func (s *PostgreSQLStore) GetMessages(
chat_id::text,
sender_id::text,
text,
created_at
created_at,
COALESCE(
ARRAY(
SELECT attachment_id::text
FROM message_attachments
WHERE message_id = messages.id
ORDER BY position
), ARRAY[]::text[]
)
FROM messages
WHERE chat_id = $1
ORDER BY
@@ -261,7 +354,15 @@ func (s *PostgreSQLStore) GetMessages(
chat_id::text,
sender_id::text,
text,
created_at
created_at,
COALESCE(
ARRAY(
SELECT attachment_id::text
FROM message_attachments
WHERE message_id = messages.id
ORDER BY position
), ARRAY[]::text[]
)
FROM messages
WHERE chat_id = $1
AND (created_at, id) > ($2, $3)
@@ -296,6 +397,7 @@ func (s *PostgreSQLStore) GetMessages(
&message.SenderID,
&message.Text,
&message.CreatedAt,
&message.AttachmentIDs,
); err != nil {
return nil, false, fmt.Errorf(
"scan message page: %w",
@@ -0,0 +1,65 @@
package application
import (
"context"
"fmt"
"io"
"mtusi2026lab4/internal/domain"
)
type AttachmentDownload struct {
Body io.ReadCloser
ContentType string
Filename string
SizeBytes int64
}
func (s *Service) GetAttachment(
ctx context.Context,
attachmentID string,
actorID string,
) (domain.Attachment, error) {
return s.repository.GetAttachment(ctx, attachmentID, actorID)
}
func (s *Service) DownloadAttachment(
ctx context.Context,
attachmentID string,
actorID string,
variant string,
) (AttachmentDownload, error) {
attachment, err := s.repository.GetAttachment(ctx, attachmentID, actorID)
if err != nil {
return AttachmentDownload{}, err
}
if attachment.Status != domain.AttachmentStatusReady {
return AttachmentDownload{}, ErrAttachmentNotReady
}
if variant == "" {
variant = "original"
}
if variant != "original" {
return AttachmentDownload{}, domain.ErrNotFound
}
if s.objectStore == nil {
return AttachmentDownload{}, ErrStorageUnavailable
}
object, err := s.objectStore.Get(ctx, attachment.ObjectKey)
if err != nil {
return AttachmentDownload{}, fmt.Errorf("%w: get object", ErrStorageUnavailable)
}
if object.Body == nil || object.SizeBytes != attachment.SizeBytes {
if object.Body != nil {
_ = object.Body.Close()
}
return AttachmentDownload{}, fmt.Errorf("%w: object metadata mismatch", ErrStorageUnavailable)
}
return AttachmentDownload{
Body: object.Body,
ContentType: attachment.ContentType,
Filename: attachment.Filename,
SizeBytes: attachment.SizeBytes,
}, nil
}
+246
View File
@@ -0,0 +1,246 @@
package application
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"os"
"mtusi2026lab4/internal/domain"
"github.com/google/uuid"
)
const attachmentPutFailedCode = "object_store_put_failed"
const attachmentFinalizeFailedCode = "metadata_finalize_failed"
func (s *Service) UploadAttachment(
ctx context.Context,
command UploadAttachmentCommand,
) (domain.Attachment, error) {
if s.objectStore == nil {
return domain.Attachment{}, ErrStorageUnavailable
}
if command.Body == nil {
return domain.Attachment{}, fmt.Errorf("%w: attachment body is required", ErrInvalidInput)
}
if err := domain.ValidateAttachmentFilename(command.Filename); err != nil {
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidInput, err)
}
if err := domain.ValidateAttachmentContentType(command.DeclaredContentType); err != nil {
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrUnsupportedMediaType, err)
}
if _, err := s.repository.GetChatByID(ctx, command.ChatID, command.ActorID); err != nil {
return domain.Attachment{}, err
}
if err := acquireAttachmentSlot(ctx, s.attachmentUploadSlots); err != nil {
return domain.Attachment{}, fmt.Errorf("%w: wait for upload capacity", ErrStorageUnavailable)
}
defer releaseAttachmentSlot(s.attachmentUploadSlots)
stagedFile, err := os.CreateTemp("", "mtusi-attachment-*")
if err != nil {
return domain.Attachment{}, fmt.Errorf("create bounded upload staging file: %w", err)
}
stagedPath := stagedFile.Name()
defer func() {
_ = stagedFile.Close()
_ = os.Remove(stagedPath)
}()
hasher := sha256.New()
limitedBody := io.LimitReader(command.Body, domain.MaxAttachmentSizeBytes+1)
sizeBytes, err := io.Copy(io.MultiWriter(stagedFile, hasher), limitedBody)
if err != nil {
return domain.Attachment{}, fmt.Errorf("%w: read attachment body", ErrInvalidInput)
}
if sizeBytes > domain.MaxAttachmentSizeBytes {
return domain.Attachment{}, ErrPayloadTooLarge
}
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
return domain.Attachment{}, fmt.Errorf("rewind staged attachment: %w", err)
}
imageConfig, format, err := image.DecodeConfig(stagedFile)
if err != nil {
return domain.Attachment{}, fmt.Errorf("%w: decode image header", ErrInvalidImage)
}
actualContentType, supported := contentTypeForImageFormat(format)
if !supported || actualContentType != command.DeclaredContentType {
return domain.Attachment{}, ErrUnsupportedMediaType
}
if imageConfig.Width < 1 || imageConfig.Height < 1 {
return domain.Attachment{}, ErrInvalidImage
}
if int64(imageConfig.Width) > int64(domain.MaxAttachmentPixels)/int64(imageConfig.Height) {
return domain.Attachment{}, ErrPayloadTooLarge
}
sha256Hex := hex.EncodeToString(hasher.Sum(nil))
if err = domain.ValidateReadyAttachmentMetadata(
command.Filename,
actualContentType,
sizeBytes,
sha256Hex,
imageConfig.Width,
imageConfig.Height,
); err != nil {
return domain.Attachment{}, fmt.Errorf("%w: %v", ErrInvalidImage, err)
}
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
return domain.Attachment{}, fmt.Errorf("rewind staged attachment for validation: %w", err)
}
if err = s.validateCompleteImage(ctx, stagedFile); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return domain.Attachment{}, fmt.Errorf("%w: wait for image decoder", ErrStorageUnavailable)
}
return domain.Attachment{}, fmt.Errorf("%w: decode complete image", ErrInvalidImage)
}
now := s.now().UTC()
attachmentID := uuid.NewString()
attachment := domain.Attachment{
ID: attachmentID,
ChatID: command.ChatID,
OwnerID: command.ActorID,
Filename: command.Filename,
ObjectKey: "attachments/" + command.ChatID + "/" + attachmentID + "/original",
ContentType: actualContentType,
SizeBytes: sizeBytes,
SHA256: sha256Hex,
Width: imageConfig.Width,
Height: imageConfig.Height,
Status: domain.AttachmentStatusPending,
Variants: make([]domain.AttachmentVariant, 0),
CreatedAt: now,
UpdatedAt: now,
}
attachment, err = s.repository.CreateAttachment(ctx, attachment)
if err != nil {
return domain.Attachment{}, err
}
if _, err = stagedFile.Seek(0, io.SeekStart); err != nil {
s.cleanupFailedAttachment(ctx, attachment, attachmentFinalizeFailedCode, false)
return domain.Attachment{}, fmt.Errorf("rewind staged attachment for storage: %w", err)
}
err = s.objectStore.Put(ctx, ObjectPut{
Key: attachment.ObjectKey,
ContentType: attachment.ContentType,
SizeBytes: attachment.SizeBytes,
Body: stagedFile,
})
if err != nil {
s.cleanupFailedAttachment(ctx, attachment, attachmentPutFailedCode, true)
return domain.Attachment{}, fmt.Errorf("%w: put object", ErrStorageUnavailable)
}
readyAttachment, err := s.repository.MarkAttachmentReady(ctx, attachment.ID, s.now().UTC())
if err != nil {
// The UPDATE may already have committed even when its response was lost.
// Deleting the object here could turn durable ready metadata into a
// permanently broken attachment. A best-effort read may recover a 201;
// otherwise both sides are preserved for the grace-period reconciler.
if resolved, ok := s.resolveAmbiguousAttachmentFinalize(ctx, attachment); ok {
return resolved, nil
}
return domain.Attachment{}, fmt.Errorf("finalize attachment metadata: %w", err)
}
if readyAttachment.Variants == nil {
readyAttachment.Variants = make([]domain.AttachmentVariant, 0)
}
return readyAttachment, nil
}
func acquireAttachmentSlot(ctx context.Context, slots chan struct{}) error {
select {
case slots <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func releaseAttachmentSlot(slots chan struct{}) {
<-slots
}
func (s *Service) validateCompleteImage(ctx context.Context, body io.Reader) error {
if err := acquireAttachmentSlot(ctx, s.attachmentDecodeSlots); err != nil {
return err
}
defer releaseAttachmentSlot(s.attachmentDecodeSlots)
_, _, err := image.Decode(body)
return err
}
func (s *Service) resolveAmbiguousAttachmentFinalize(
requestContext context.Context,
attachment domain.Attachment,
) (domain.Attachment, bool) {
resolveContext, cancel := context.WithTimeout(
context.WithoutCancel(requestContext),
s.attachmentCleanupTimeout,
)
defer cancel()
stored, err := s.repository.GetAttachment(
resolveContext,
attachment.ID,
attachment.OwnerID,
)
if err != nil || stored.Status != domain.AttachmentStatusReady {
return domain.Attachment{}, false
}
if stored.Variants == nil {
stored.Variants = make([]domain.AttachmentVariant, 0)
}
return stored, true
}
func contentTypeForImageFormat(format string) (string, bool) {
switch format {
case "png":
return "image/png", true
case "jpeg":
return "image/jpeg", true
default:
return "", false
}
}
func (s *Service) cleanupFailedAttachment(
requestContext context.Context,
attachment domain.Attachment,
errorCode string,
deleteObject bool,
) {
metadataContext, metadataCancel := context.WithTimeout(
context.WithoutCancel(requestContext),
s.attachmentCleanupTimeout,
)
_ = s.repository.MarkAttachmentFailed(
metadataContext,
attachment.ID,
errorCode,
s.now().UTC(),
)
metadataCancel()
if !deleteObject {
return
}
deleteContext, deleteCancel := context.WithTimeout(
context.WithoutCancel(requestContext),
s.attachmentCleanupTimeout,
)
defer deleteCancel()
_ = s.objectStore.Delete(deleteContext, attachment.ObjectKey)
}
+524
View File
@@ -0,0 +1,524 @@
package application_test
import (
"bytes"
"context"
"encoding/binary"
"errors"
"hash/crc32"
"image"
"image/color"
"image/jpeg"
"image/png"
"io"
"strings"
"sync"
"testing"
"time"
"mtusi2026lab4/internal/adapter/memory"
. "mtusi2026lab4/internal/application"
"mtusi2026lab4/internal/domain"
)
type fakeObjectStore struct {
mu sync.Mutex
objects map[string][]byte
putError error
blockDelete bool
putCalls int
deleteCalls int
lastPutKey string
}
type alwaysFailReader struct{}
func (alwaysFailReader) Read([]byte) (int, error) {
return 0, errors.New("injected interrupted upload")
}
type finalizeFailureRepository struct {
Repository
}
func (repository *finalizeFailureRepository) MarkAttachmentReady(
context.Context,
string,
time.Time,
) (domain.Attachment, error) {
return domain.Attachment{}, errors.New("injected metadata finalize failure")
}
type commitThenDisconnectRepository struct {
Repository
hideReadback bool
}
func (repository *commitThenDisconnectRepository) MarkAttachmentReady(
ctx context.Context,
attachmentID string,
updatedAt time.Time,
) (domain.Attachment, error) {
if _, err := repository.Repository.MarkAttachmentReady(ctx, attachmentID, updatedAt); err != nil {
return domain.Attachment{}, err
}
return domain.Attachment{}, errors.New("injected connection loss after commit")
}
func (repository *commitThenDisconnectRepository) GetAttachment(
ctx context.Context,
attachmentID string,
actorID string,
) (domain.Attachment, error) {
if repository.hideReadback {
return domain.Attachment{}, errors.New("injected database outage during readback")
}
return repository.Repository.GetAttachment(ctx, attachmentID, actorID)
}
func newFakeObjectStore() *fakeObjectStore {
return &fakeObjectStore{objects: make(map[string][]byte)}
}
func (store *fakeObjectStore) Put(_ context.Context, object ObjectPut) error {
store.mu.Lock()
defer store.mu.Unlock()
store.putCalls++
store.lastPutKey = object.Key
if store.putError != nil {
return store.putError
}
payload, err := io.ReadAll(object.Body)
if err != nil {
return err
}
if int64(len(payload)) != object.SizeBytes {
return errors.New("declared object size differs")
}
store.objects[object.Key] = payload
return nil
}
func (store *fakeObjectStore) Get(_ context.Context, key string) (ObjectContent, error) {
store.mu.Lock()
defer store.mu.Unlock()
payload, exists := store.objects[key]
if !exists {
return ObjectContent{}, errors.New("object not found")
}
copyOfPayload := append([]byte(nil), payload...)
return ObjectContent{
Body: io.NopCloser(bytes.NewReader(copyOfPayload)),
SizeBytes: int64(len(copyOfPayload)),
}, nil
}
func (store *fakeObjectStore) Delete(ctx context.Context, key string) error {
store.mu.Lock()
store.deleteCalls++
blockDelete := store.blockDelete
store.mu.Unlock()
if blockDelete {
<-ctx.Done()
return ctx.Err()
}
store.mu.Lock()
defer store.mu.Unlock()
delete(store.objects, key)
return nil
}
func newAttachmentService(
t *testing.T,
objectStore ObjectStore,
) (*Service, *memory.MemoryStore, string, string) {
t.Helper()
legacyStore := memory.NewMemoryStore()
repository := memory.NewMemoryRepository(legacyStore)
user, err := repository.CreateUser(context.Background(), "uploader", "Uploader", "hash")
if err != nil {
t.Fatal(err)
}
chat, err := repository.CreateChat(context.Background(), "Images", []string{user.ID})
if err != nil {
t.Fatal(err)
}
codec, err := NewCursorCodec("attachment-test-key-with-more-than-thirty-two-bytes")
if err != nil {
t.Fatal(err)
}
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
if err != nil {
t.Fatal(err)
}
return service, legacyStore, user.ID, chat.ID
}
func encodeTestImage(t *testing.T, contentType string) []byte {
t.Helper()
var payload bytes.Buffer
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
img.Set(0, 0, color.RGBA{R: 255, A: 255})
var err error
if contentType == "image/png" {
err = png.Encode(&payload, img)
} else {
err = jpeg.Encode(&payload, img, nil)
}
if err != nil {
t.Fatal(err)
}
return payload.Bytes()
}
func corruptPNGAfterPrefix(t *testing.T) []byte {
t.Helper()
img := image.NewNRGBA(image.Rect(0, 0, 512, 512))
state := uint32(0x12345678)
for index := range img.Pix {
state ^= state << 13
state ^= state >> 17
state ^= state << 5
img.Pix[index] = byte(state)
}
var payload bytes.Buffer
if err := png.Encode(&payload, img); err != nil {
t.Fatal(err)
}
corrupted := payload.Bytes()
if len(corrupted) <= 70_000 {
t.Fatalf("test PNG is unexpectedly small: %d", len(corrupted))
}
corrupted[70_000] ^= 0xff
return corrupted
}
func jpegWithoutScanData(t *testing.T) []byte {
t.Helper()
payload := encodeTestImage(t, "image/jpeg")
sos := bytes.Index(payload, []byte{0xff, 0xda})
if sos < 0 || sos+4 > len(payload) {
t.Fatal("encoded JPEG has no SOS marker")
}
segmentLength := int(binary.BigEndian.Uint16(payload[sos+2 : sos+4]))
endOfHeader := sos + 2 + segmentLength
if endOfHeader > len(payload) {
t.Fatal("invalid SOS segment length in test JPEG")
}
return append([]byte(nil), payload[:endOfHeader]...)
}
func TestUploadAttachmentStoresValidatedImage(t *testing.T) {
objectStore := newFakeObjectStore()
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
payload := encodeTestImage(t, "image/png")
attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID,
ChatID: chatID,
Filename: "кот.png",
DeclaredContentType: "image/png",
Body: bytes.NewReader(payload),
})
if err != nil {
t.Fatal(err)
}
if attachment.Status != domain.AttachmentStatusReady {
t.Fatalf("status = %q, want ready", attachment.Status)
}
if attachment.ObjectKey == "" || strings.Contains(attachment.ObjectKey, attachment.Filename) {
t.Fatalf("object key must be server-generated: %q", attachment.ObjectKey)
}
if attachment.Width != 2 || attachment.Height != 2 || len(attachment.SHA256) != 64 {
t.Fatalf("unexpected image metadata: %+v", attachment)
}
if !bytes.Equal(objectStore.objects[attachment.ObjectKey], payload) {
t.Fatal("stored object differs from uploaded bytes")
}
stored, err := metadataStore.GetAttachmentByID(attachment.ID)
if err != nil || stored.Status != domain.AttachmentStatusReady {
t.Fatalf("ready metadata was not persisted: %+v, %v", stored, err)
}
}
func TestUploadAttachmentRejectsInvalidInputBeforeStorage(t *testing.T) {
testCases := []struct {
name string
filename string
declaredType string
payload []byte
wantError error
}{
{name: "unsafe filename", filename: "../image.png", declaredType: "image/png", payload: encodeTestImage(t, "image/png"), wantError: ErrInvalidInput},
{name: "unsupported declared type", filename: "image.gif", declaredType: "image/gif", payload: []byte("GIF89a"), wantError: ErrUnsupportedMediaType},
{name: "MIME substitution", filename: "image.jpg", declaredType: "image/jpeg", payload: encodeTestImage(t, "image/png"), wantError: ErrUnsupportedMediaType},
{name: "corrupt image", filename: "image.png", declaredType: "image/png", payload: []byte("not a png"), wantError: ErrInvalidImage},
{name: "PNG header without image data", filename: "truncated.png", declaredType: "image/png", payload: pngConfigurationOnly(1, 1), wantError: ErrInvalidImage},
{name: "PNG corruption after 64 KiB", filename: "corrupted.png", declaredType: "image/png", payload: corruptPNGAfterPrefix(t), wantError: ErrInvalidImage},
{name: "JPEG without scan data", filename: "truncated.jpg", declaredType: "image/jpeg", payload: jpegWithoutScanData(t), wantError: ErrInvalidImage},
{name: "too large", filename: "image.png", declaredType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantError: ErrPayloadTooLarge},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
objectStore := newFakeObjectStore()
service, _, actorID, chatID := newAttachmentService(t, objectStore)
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID,
ChatID: chatID,
Filename: testCase.filename,
DeclaredContentType: testCase.declaredType,
Body: bytes.NewReader(testCase.payload),
})
if !errors.Is(err, testCase.wantError) {
t.Fatalf("error = %v, want %v", err, testCase.wantError)
}
if objectStore.putCalls != 0 {
t.Fatalf("invalid upload reached object storage %d times", objectStore.putCalls)
}
})
}
}
func TestUploadAttachmentCompensatesObjectStoreFailure(t *testing.T) {
objectStore := newFakeObjectStore()
objectStore.putError = errors.New("injected put failure")
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID,
ChatID: chatID,
Filename: "image.jpg",
DeclaredContentType: "image/jpeg",
Body: bytes.NewReader(encodeTestImage(t, "image/jpeg")),
})
if !errors.Is(err, ErrStorageUnavailable) {
t.Fatalf("error = %v, want ErrStorageUnavailable", err)
}
if objectStore.deleteCalls != 1 {
t.Fatalf("compensating deletes = %d, want 1", objectStore.deleteCalls)
}
objectStore.mu.Lock()
keyParts := strings.Split(objectStore.lastPutKey, "/")
objectStore.mu.Unlock()
if len(keyParts) != 4 {
t.Fatalf("unexpected generated object key: %q", objectStore.lastPutKey)
}
failedAttachment, lookupErr := metadataStore.GetAttachmentByID(keyParts[2])
if lookupErr != nil {
t.Fatal(lookupErr)
}
if failedAttachment.Status != domain.AttachmentStatusFailed || failedAttachment.ErrorCode == nil {
t.Fatalf("failed upload metadata was not finalized: %+v", failedAttachment)
}
}
func TestUploadAttachmentMarksFailedBeforeBlockedObjectCleanup(t *testing.T) {
objectStore := newFakeObjectStore()
objectStore.putError = errors.New("injected put failure")
objectStore.blockDelete = true
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
startedAt := time.Now()
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID, ChatID: chatID, Filename: "blocked-cleanup.png",
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
})
if !errors.Is(err, ErrStorageUnavailable) {
t.Fatalf("error = %v, want ErrStorageUnavailable", err)
}
if elapsed := time.Since(startedAt); elapsed > 2*time.Second {
t.Fatalf("bounded cleanup took %s", elapsed)
}
keyParts := strings.Split(objectStore.lastPutKey, "/")
failed, lookupErr := metadataStore.GetAttachmentByID(keyParts[2])
if lookupErr != nil {
t.Fatal(lookupErr)
}
if failed.Status != domain.AttachmentStatusFailed || failed.ErrorCode == nil {
t.Fatalf("blocked object cleanup consumed DB cleanup context: %+v", failed)
}
}
func TestUploadAttachmentRejectsInterruptedBodyBeforeMetadata(t *testing.T) {
objectStore := newFakeObjectStore()
service, metadataStore, actorID, chatID := newAttachmentService(t, objectStore)
payload := encodeTestImage(t, "image/png")
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID,
ChatID: chatID,
Filename: "interrupted.png",
DeclaredContentType: "image/png",
Body: io.MultiReader(bytes.NewReader(payload[:8]), alwaysFailReader{}),
})
if !errors.Is(err, ErrInvalidInput) {
t.Fatalf("error = %v, want ErrInvalidInput", err)
}
if objectStore.putCalls != 0 {
t.Fatal("interrupted upload reached object storage")
}
if _, lookupErr := metadataStore.GetAttachmentByID("missing"); !errors.Is(lookupErr, domain.ErrNotFound) {
t.Fatalf("unexpected metadata lookup result: %v", lookupErr)
}
}
func TestUploadAttachmentPreservesObjectOnMetadataFinalizeFailure(t *testing.T) {
legacyStore := memory.NewMemoryStore()
repository := memory.NewMemoryRepository(legacyStore)
user, err := repository.CreateUser(context.Background(), "finalize", "Finalize", "hash")
if err != nil {
t.Fatal(err)
}
chat, err := repository.CreateChat(context.Background(), "Finalize", []string{user.ID})
if err != nil {
t.Fatal(err)
}
codec, err := NewCursorCodec("finalize-test-key-with-more-than-thirty-two-bytes")
if err != nil {
t.Fatal(err)
}
objectStore := newFakeObjectStore()
service, err := NewService(
&finalizeFailureRepository{Repository: repository},
codec,
time.Hour,
WithObjectStore(objectStore),
)
if err != nil {
t.Fatal(err)
}
_, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: user.ID,
ChatID: chat.ID,
Filename: "finalize.png",
DeclaredContentType: "image/png",
Body: bytes.NewReader(encodeTestImage(t, "image/png")),
})
if err == nil {
t.Fatal("metadata finalize failure was hidden")
}
if objectStore.deleteCalls != 0 || len(objectStore.objects) != 1 {
t.Fatalf("ambiguous object must be preserved: deletes=%d objects=%d", objectStore.deleteCalls, len(objectStore.objects))
}
keyParts := strings.Split(objectStore.lastPutKey, "/")
pending, lookupErr := legacyStore.GetAttachmentByID(keyParts[2])
if lookupErr != nil {
t.Fatal(lookupErr)
}
if pending.Status != domain.AttachmentStatusPending || pending.ErrorCode != nil {
t.Fatalf("ambiguous metadata must remain reconcilable: %+v", pending)
}
}
func TestUploadAttachmentRecoversWhenCommitResponseIsLost(t *testing.T) {
legacyStore := memory.NewMemoryStore()
baseRepository := memory.NewMemoryRepository(legacyStore)
user, err := baseRepository.CreateUser(context.Background(), "commit-lost", "Commit Lost", "hash")
if err != nil {
t.Fatal(err)
}
chat, err := baseRepository.CreateChat(context.Background(), "Commit Lost", []string{user.ID})
if err != nil {
t.Fatal(err)
}
codec, err := NewCursorCodec("commit-lost-test-key-with-more-than-thirty-two-bytes")
if err != nil {
t.Fatal(err)
}
objectStore := newFakeObjectStore()
repository := &commitThenDisconnectRepository{Repository: baseRepository}
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
if err != nil {
t.Fatal(err)
}
attachment, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png",
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
})
if err != nil {
t.Fatalf("committed ready attachment was not recovered: %v", err)
}
if attachment.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 {
t.Fatalf("lost commit response damaged attachment: attachment=%+v deletes=%d objects=%d",
attachment, objectStore.deleteCalls, len(objectStore.objects))
}
}
func TestUploadAttachmentNeverDeletesObjectAfterAmbiguousCommit(t *testing.T) {
legacyStore := memory.NewMemoryStore()
baseRepository := memory.NewMemoryRepository(legacyStore)
user, err := baseRepository.CreateUser(context.Background(), "commit-outage", "Commit Outage", "hash")
if err != nil {
t.Fatal(err)
}
chat, err := baseRepository.CreateChat(context.Background(), "Commit Outage", []string{user.ID})
if err != nil {
t.Fatal(err)
}
codec, err := NewCursorCodec("commit-outage-test-key-with-more-than-thirty-two-bytes")
if err != nil {
t.Fatal(err)
}
objectStore := newFakeObjectStore()
repository := &commitThenDisconnectRepository{Repository: baseRepository, hideReadback: true}
service, err := NewService(repository, codec, time.Hour, WithObjectStore(objectStore))
if err != nil {
t.Fatal(err)
}
_, err = service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: user.ID, ChatID: chat.ID, Filename: "commit.png",
DeclaredContentType: "image/png", Body: bytes.NewReader(encodeTestImage(t, "image/png")),
})
if err == nil {
t.Fatal("database outage after commit was hidden")
}
keyParts := strings.Split(objectStore.lastPutKey, "/")
stored, lookupErr := legacyStore.GetAttachmentByID(keyParts[2])
if lookupErr != nil {
t.Fatal(lookupErr)
}
if stored.Status != domain.AttachmentStatusReady || len(objectStore.objects) != 1 || objectStore.deleteCalls != 0 {
t.Fatalf("committed ready metadata lost its original: metadata=%+v deletes=%d objects=%d",
stored, objectStore.deleteCalls, len(objectStore.objects))
}
}
func TestUploadAttachmentRejectsPixelLimitBeforeFullDecode(t *testing.T) {
objectStore := newFakeObjectStore()
service, _, actorID, chatID := newAttachmentService(t, objectStore)
_, err := service.UploadAttachment(context.Background(), UploadAttachmentCommand{
ActorID: actorID,
ChatID: chatID,
Filename: "huge.png",
DeclaredContentType: "image/png",
Body: bytes.NewReader(pngConfigurationOnly(10_000, 4_001)),
})
if !errors.Is(err, ErrPayloadTooLarge) {
t.Fatalf("error = %v, want ErrPayloadTooLarge", err)
}
if objectStore.putCalls != 0 {
t.Fatalf("oversized decoded image reached object storage %d times", objectStore.putCalls)
}
}
func pngConfigurationOnly(width uint32, height uint32) []byte {
payload := make([]byte, 33)
copy(payload[0:8], []byte{137, 80, 78, 71, 13, 10, 26, 10})
binary.BigEndian.PutUint32(payload[8:12], 13)
copy(payload[12:16], "IHDR")
binary.BigEndian.PutUint32(payload[16:20], width)
binary.BigEndian.PutUint32(payload[20:24], height)
payload[24] = 8
payload[25] = 6
payload[26] = 0
payload[27] = 0
payload[28] = 0
binary.BigEndian.PutUint32(payload[29:33], crc32.ChecksumIEEE(payload[12:29]))
return payload
}
+11
View File
@@ -1,5 +1,7 @@
package application
import "io"
type CreateUserCommand struct {
Username string
DisplayName string
@@ -21,7 +23,16 @@ type SendMessageCommand struct {
ActorID string
ChatID string
Text string
AttachmentIDs []string
Method string
CanonicalPath string
IdempotencyKey string
}
type UploadAttachmentCommand struct {
ActorID string
ChatID string
Filename string
DeclaredContentType string
Body io.Reader
}
+30 -4
View File
@@ -1,7 +1,9 @@
package application
import (
"encoding/base64"
"errors"
"strings"
"testing"
"time"
@@ -47,11 +49,13 @@ func TestCursorCodecRejectsTamperingAndWrongScope(t *testing.T) {
t.Fatal(err)
}
replacement := "A"
if token[len(token)-1:] == replacement {
replacement = "B"
parts := strings.Split(token, ".")
signature, err := base64.RawURLEncoding.DecodeString(parts[1])
if err != nil {
t.Fatal(err)
}
tampered := token[:len(token)-1] + replacement
signature[0] ^= 1
tampered := parts[0] + "." + base64.RawURLEncoding.EncodeToString(signature)
if _, err = codec.DecodeChat(tampered, "9fa3f49c-e62e-4209-8414-d6d5f69805bf"); !errors.Is(err, ErrInvalidCursor) {
t.Fatalf("tampered cursor: got %v, want ErrInvalidCursor", err)
}
@@ -68,3 +72,25 @@ func TestCursorCodecRequiresStrongSharedKey(t *testing.T) {
t.Fatal("expected short cursor signing key to be rejected")
}
}
func TestCursorCodecRejectsSignedInvalidFieldTypesBeforeSQL(t *testing.T) {
codec, err := NewCursorCodec(cursorTestKey)
if err != nil {
t.Fatal(err)
}
chatID := "9fa3f49c-e62e-4209-8414-d6d5f69805bf"
validMessageID := "42eea32e-d9b5-4bc6-b4fc-94e9b0477adf"
testCases := []cursorPayload{
{Version: cursorVersion, Kind: "message", CreatedAt: "not-a-time", ID: validMessageID, ScopeID: chatID},
{Version: cursorVersion, Kind: "message", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), ID: "not-a-uuid", ScopeID: chatID},
}
for _, payload := range testCases {
token, encodeErr := codec.encode(payload)
if encodeErr != nil {
t.Fatal(encodeErr)
}
if _, decodeErr := codec.DecodeMessage(token, chatID); !errors.Is(decodeErr, ErrInvalidCursor) {
t.Fatalf("invalid signed cursor returned %v, want ErrInvalidCursor", decodeErr)
}
}
}
+10 -5
View File
@@ -3,9 +3,14 @@ package application
import "errors"
var (
ErrInvalidCursor = errors.New("invalid cursor")
ErrInvalidCredentials = errors.New("invalid credentials")
ErrInvalidInput = errors.New("invalid input")
ErrCredentialState = errors.New("invalid stored credential state")
ErrIdempotencyConflict = errors.New("idempotency key is already used for another request")
ErrInvalidCursor = errors.New("invalid cursor")
ErrInvalidCredentials = errors.New("invalid credentials")
ErrInvalidInput = errors.New("invalid input")
ErrCredentialState = errors.New("invalid stored credential state")
ErrIdempotencyConflict = errors.New("idempotency key is already used for another request")
ErrAttachmentNotReady = errors.New("attachment is not ready")
ErrPayloadTooLarge = errors.New("payload too large")
ErrUnsupportedMediaType = errors.New("unsupported media type")
ErrInvalidImage = errors.New("invalid image")
ErrStorageUnavailable = errors.New("object storage unavailable")
)
+13 -2
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"mtusi2026lab4/internal/domain"
"github.com/google/uuid"
)
type canonicalMessageRequest struct {
@@ -15,13 +17,21 @@ type canonicalMessageRequest struct {
}
func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) (domain.Message, error) {
if err := domain.ValidateMessageText(command.Text); err != nil {
attachmentIDs := append([]string(nil), command.AttachmentIDs...)
if attachmentIDs == nil {
attachmentIDs = make([]string, 0)
}
if err := domain.ValidateMessageInput(command.Text, attachmentIDs); err != nil {
return domain.Message{}, fmt.Errorf("%w: %v", ErrInvalidInput, err)
}
for index, attachmentID := range attachmentIDs {
attachmentIDs[index] = uuid.MustParse(attachmentID).String()
}
canonicalBody, err := json.Marshal(canonicalMessageRequest{
Text: command.Text,
AttachmentIDs: make([]string, 0),
AttachmentIDs: attachmentIDs,
})
if err != nil {
return domain.Message{}, fmt.Errorf("canonicalize message request: %w", err)
@@ -31,6 +41,7 @@ func (s *Service) SendMessage(ctx context.Context, command SendMessageCommand) (
ActorID: command.ActorID,
ChatID: command.ChatID,
Text: command.Text,
AttachmentIDs: attachmentIDs,
Method: command.Method,
CanonicalPath: command.CanonicalPath,
Key: command.IdempotencyKey,
+28
View File
@@ -2,6 +2,7 @@ package application
import (
"context"
"io"
"time"
"mtusi2026lab4/internal/domain"
@@ -28,6 +29,7 @@ type IdempotentMessageWrite struct {
ActorID string
ChatID string
Text string
AttachmentIDs []string
Method string
CanonicalPath string
Key string
@@ -40,9 +42,35 @@ type MessageRepository interface {
GetMessages(ctx context.Context, chatID, callerID string, limit int, cursor *domain.MessageCursor) ([]domain.Message, bool, error)
}
type AttachmentRepository interface {
CreateAttachment(ctx context.Context, attachment domain.Attachment) (domain.Attachment, error)
MarkAttachmentReady(ctx context.Context, attachmentID string, updatedAt time.Time) (domain.Attachment, error)
MarkAttachmentFailed(ctx context.Context, attachmentID, errorCode string, updatedAt time.Time) error
GetAttachment(ctx context.Context, attachmentID, callerID string) (domain.Attachment, error)
}
type Repository interface {
UserRepository
SessionRepository
ChatRepository
MessageRepository
AttachmentRepository
}
type ObjectPut struct {
Key string
ContentType string
SizeBytes int64
Body io.Reader
}
type ObjectContent struct {
Body io.ReadCloser
SizeBytes int64
}
type ObjectStore interface {
Put(ctx context.Context, object ObjectPut) error
Get(ctx context.Context, key string) (ObjectContent, error)
Delete(ctx context.Context, key string) error
}
+30 -10
View File
@@ -6,15 +6,22 @@ import (
)
const MinimumIdempotencyTTL = 24 * time.Hour
const defaultAttachmentCleanupTimeout = time.Second
const defaultAttachmentUploadConcurrency = 4
const defaultAttachmentDecodeConcurrency = 1
type ServiceOption func(*Service) error
type Service struct {
repository Repository
cursors *CursorCodec
sessionTTL time.Duration
idempotencyTTL time.Duration
now func() time.Time
repository Repository
cursors *CursorCodec
sessionTTL time.Duration
idempotencyTTL time.Duration
now func() time.Time
objectStore ObjectStore
attachmentCleanupTimeout time.Duration
attachmentUploadSlots chan struct{}
attachmentDecodeSlots chan struct{}
}
func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Duration, options ...ServiceOption) (*Service, error) {
@@ -28,11 +35,14 @@ func NewService(repository Repository, cursors *CursorCodec, sessionTTL time.Dur
return nil, fmt.Errorf("session TTL must be positive")
}
service := &Service{
repository: repository,
cursors: cursors,
sessionTTL: sessionTTL,
idempotencyTTL: MinimumIdempotencyTTL,
now: func() time.Time { return time.Now().UTC() },
repository: repository,
cursors: cursors,
sessionTTL: sessionTTL,
idempotencyTTL: MinimumIdempotencyTTL,
now: func() time.Time { return time.Now().UTC() },
attachmentCleanupTimeout: defaultAttachmentCleanupTimeout,
attachmentUploadSlots: make(chan struct{}, defaultAttachmentUploadConcurrency),
attachmentDecodeSlots: make(chan struct{}, defaultAttachmentDecodeConcurrency),
}
for _, option := range options {
if err := option(service); err != nil {
@@ -51,3 +61,13 @@ func WithIdempotencyTTL(ttl time.Duration) ServiceOption {
return nil
}
}
func WithObjectStore(objectStore ObjectStore) ServiceOption {
return func(service *Service) error {
if objectStore == nil {
return fmt.Errorf("object store is required")
}
service.objectStore = objectStore
return nil
}
}
+31
View File
@@ -35,6 +35,13 @@ type Config struct {
RedisOperationTimeout time.Duration
RedisPoolSize int
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKeyID string
S3SecretAccessKey string
S3OperationTimeout time.Duration
ReadinessTimeout time.Duration
ShutdownDrainDelay time.Duration
ShutdownTimeout time.Duration
@@ -82,6 +89,24 @@ func Load() (Config, error) {
return Config{}, err
}
s3AccessKeyID, err := requiredValueOrFile(
"S3_ACCESS_KEY_ID",
"S3_ACCESS_KEY_ID_FILE",
)
if err != nil {
return Config{}, err
}
s3SecretAccessKey, err := requiredValueOrFile(
"S3_SECRET_ACCESS_KEY",
"S3_SECRET_ACCESS_KEY_FILE",
)
if err != nil {
return Config{}, err
}
s3OperationTimeout, err := durationFromEnv("S3_OPERATION_TIMEOUT", 5*time.Second)
if err != nil {
return Config{}, err
}
databaseMaxConns, err := int32FromEnv("DB_MAX_CONNS", 10, 1)
if err != nil {
return Config{}, err
@@ -223,6 +248,12 @@ func Load() (Config, error) {
RedisOperationTimeout: redisOperationTimeout,
RedisPoolSize: int(redisPoolSize),
S3Endpoint: envOrDefault("S3_ENDPOINT", "http://minio:9000"),
S3Region: envOrDefault("S3_REGION", "us-east-1"),
S3Bucket: envOrDefault("S3_BUCKET", "messenger-attachments"),
S3AccessKeyID: s3AccessKeyID,
S3SecretAccessKey: s3SecretAccessKey,
S3OperationTimeout: s3OperationTimeout,
ReadinessTimeout: readinessTimeout,
ShutdownDrainDelay: shutdownDrainDelay,
ShutdownTimeout: shutdownTimeout,
+119
View File
@@ -0,0 +1,119 @@
package domain
import (
"encoding/hex"
"fmt"
"strings"
"time"
"unicode"
"unicode/utf8"
)
const (
MaxAttachmentFilenameCodePoints = 255
MaxAttachmentSizeBytes = 10 * 1024 * 1024
MaxAttachmentPixels = 40_000_000
)
type AttachmentVariant struct {
Name string
ContentType string
SizeBytes int64
Width int
Height int
SHA256 string
}
type AttachmentStatus string
const (
AttachmentStatusPending AttachmentStatus = "pending"
AttachmentStatusReady AttachmentStatus = "ready"
AttachmentStatusFailed AttachmentStatus = "failed"
)
// Attachment contains metadata only. ObjectKey points to bytes in the
// private object store and must never be derived from Filename.
type Attachment struct {
ID string
ChatID string
OwnerID string
Filename string
ObjectKey string
ContentType string
SizeBytes int64
SHA256 string
Width int
Height int
Status AttachmentStatus
Variants []AttachmentVariant
ErrorCode *string
CreatedAt time.Time
UpdatedAt time.Time
}
func ValidateAttachmentFilename(filename string) error {
if !utf8.ValidString(filename) {
return fmt.Errorf("filename must be valid UTF-8")
}
filenameLength := utf8.RuneCountInString(filename)
if filenameLength < 1 || filenameLength > MaxAttachmentFilenameCodePoints {
return fmt.Errorf("filename length must be between 1 and %d code points", MaxAttachmentFilenameCodePoints)
}
if strings.TrimSpace(filename) == "" || filename == "." || filename == ".." {
return fmt.Errorf("filename is unsafe")
}
if strings.ContainsAny(filename, `/\\`) {
return fmt.Errorf("filename must not contain path separators")
}
for _, character := range filename {
if unicode.IsControl(character) {
return fmt.Errorf("filename must not contain control characters")
}
}
return nil
}
func ValidateAttachmentContentType(contentType string) error {
if contentType != "image/png" && contentType != "image/jpeg" {
return fmt.Errorf("content type must be image/png or image/jpeg")
}
return nil
}
// ValidateReadyAttachmentMetadata checks the bounded metadata produced after
// the upload stream has been identified, decoded and hashed. It deliberately
// does not trust an HTTP Content-Type header.
func ValidateReadyAttachmentMetadata(
filename string,
contentType string,
sizeBytes int64,
sha256Hex string,
width int,
height int,
) error {
if err := ValidateAttachmentFilename(filename); err != nil {
return err
}
if err := ValidateAttachmentContentType(contentType); err != nil {
return err
}
if sizeBytes < 1 || sizeBytes > MaxAttachmentSizeBytes {
return fmt.Errorf("attachment size must be between 1 and %d bytes", MaxAttachmentSizeBytes)
}
if width < 1 || height < 1 {
return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels)
}
if int64(width) > int64(MaxAttachmentPixels)/int64(height) {
return fmt.Errorf("image dimensions must be positive and contain at most %d pixels", MaxAttachmentPixels)
}
if len(sha256Hex) != 64 || sha256Hex != strings.ToLower(sha256Hex) {
return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters")
}
decodedHash, err := hex.DecodeString(sha256Hex)
if err != nil || len(decodedHash) != 32 {
return fmt.Errorf("sha256 must contain 64 lowercase hexadecimal characters")
}
return nil
}
+59
View File
@@ -0,0 +1,59 @@
package domain
import (
"math"
"strings"
"testing"
)
func TestValidateReadyAttachmentMetadata(t *testing.T) {
validHash := strings.Repeat("ab", 32)
testCases := []struct {
name string
filename string
contentType string
sizeBytes int64
sha256 string
width int
height int
wantError bool
}{
{name: "png", filename: "кот.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1},
{name: "jpeg at limits", filename: strings.Repeat("界", 255), contentType: "image/jpeg", sizeBytes: MaxAttachmentSizeBytes, sha256: validHash, width: 8_000, height: 5_000},
{name: "empty filename", filename: "", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "blank filename", filename: " ", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "dot segment", filename: "..", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "forward slash", filename: "folder/image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "backslash", filename: `folder\\image.png`, contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "control character", filename: "image\n.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "C1 control character", filename: "image\u0085.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "invalid UTF-8", filename: string([]byte{'x', 0xff, '.', 'p', 'n', 'g'}), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "filename too long", filename: strings.Repeat("a", 256), contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "unsupported MIME", filename: "image.gif", contentType: "image/gif", sizeBytes: 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "empty file", filename: "image.png", contentType: "image/png", sizeBytes: 0, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "file too large", filename: "image.png", contentType: "image/png", sizeBytes: MaxAttachmentSizeBytes + 1, sha256: validHash, width: 1, height: 1, wantError: true},
{name: "zero width", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 0, height: 1, wantError: true},
{name: "too many pixels", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: 8_001, height: 5_000, wantError: true},
{name: "dimension multiplication cannot overflow", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: validHash, width: math.MaxInt, height: math.MaxInt, wantError: true},
{name: "short hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: "ab", width: 1, height: 1, wantError: true},
{name: "uppercase hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.ToUpper(validHash), width: 1, height: 1, wantError: true},
{name: "non-hex hash", filename: "image.png", contentType: "image/png", sizeBytes: 1, sha256: strings.Repeat("zz", 32), width: 1, height: 1, wantError: true},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := ValidateReadyAttachmentMetadata(
testCase.filename,
testCase.contentType,
testCase.sizeBytes,
testCase.sha256,
testCase.width,
testCase.height,
)
if (err != nil) != testCase.wantError {
t.Fatalf("ValidateReadyAttachmentMetadata() error = %v, wantError %v", err, testCase.wantError)
}
})
}
}
+6 -5
View File
@@ -3,11 +3,12 @@ package domain
import "time"
type Message struct {
ID string
ChatID string
SenderID string
Text string
CreatedAt time.Time
ID string
ChatID string
SenderID string
Text string
AttachmentIDs []string
CreatedAt time.Time
}
type MessageCursor struct {
+70
View File
@@ -0,0 +1,70 @@
package domain
import (
"strings"
"testing"
)
func TestValidateMessageInput(t *testing.T) {
validAttachment := "11111111-1111-4111-8111-111111111111"
testCases := []struct {
name string
text string
attachmentIDs []string
wantError bool
}{
{name: "text only", text: "hello"},
{name: "attachment only", attachmentIDs: []string{validAttachment}},
{name: "whitespace with attachment", text: " ", attachmentIDs: []string{validAttachment}},
{name: "empty", wantError: true},
{name: "whitespace only", text: " ", wantError: true},
{name: "text too long", text: strings.Repeat("界", 4001), wantError: true},
{name: "invalid attachment uuid", text: "hello", attachmentIDs: []string{"not-a-uuid"}, wantError: true},
{
name: "duplicate attachment uuid ignores case",
text: "hello",
attachmentIDs: []string{
"AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA",
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
},
wantError: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := ValidateMessageInput(testCase.text, testCase.attachmentIDs)
if (err != nil) != testCase.wantError {
t.Fatalf("ValidateMessageInput() error = %v, wantError %v", err, testCase.wantError)
}
})
}
}
func TestValidateMessageInputAttachmentLimit(t *testing.T) {
attachmentIDs := []string{
"00000000-0000-4000-8000-000000000001",
"00000000-0000-4000-8000-000000000002",
"00000000-0000-4000-8000-000000000003",
"00000000-0000-4000-8000-000000000004",
"00000000-0000-4000-8000-000000000005",
"00000000-0000-4000-8000-000000000006",
"00000000-0000-4000-8000-000000000007",
"00000000-0000-4000-8000-000000000008",
"00000000-0000-4000-8000-000000000009",
"00000000-0000-4000-8000-000000000010",
}
if err := ValidateMessageInput("", attachmentIDs); err != nil {
t.Fatalf("10 attachments must be accepted: %v", err)
}
attachmentIDs = append(
attachmentIDs,
"00000000-0000-4000-8000-000000000011",
)
if err := ValidateMessageInput("hello", attachmentIDs); err == nil {
t.Fatal("11 attachments must be rejected")
}
}
+29 -5
View File
@@ -9,6 +9,8 @@ import (
"github.com/google/uuid"
)
const MaxMessageAttachments = 10
var usernameRegex = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
func ValidateUser(username, displayName string) error {
@@ -65,13 +67,35 @@ func ValidateChatInput(title string, memberIDs []string, callerID string) error
return nil
}
func ValidateMessageText(text string) error {
func ValidateMessageInput(text string, attachmentIDs []string) error {
textRunes := utf8.RuneCountInString(text)
if textRunes < 1 || textRunes > 4000 {
return fmt.Errorf("message text length must be between 1 and 4000 code points")
if textRunes > 4000 {
return fmt.Errorf("message text length must be at most 4000 code points")
}
if strings.TrimSpace(text) == "" {
return fmt.Errorf("message text cannot be only whitespace")
if len(attachmentIDs) > MaxMessageAttachments {
return fmt.Errorf("attachment_ids must contain at most %d items", MaxMessageAttachments)
}
seen := make(map[string]struct{}, len(attachmentIDs))
for _, attachmentID := range attachmentIDs {
parsedID, err := uuid.Parse(attachmentID)
if err != nil {
return fmt.Errorf("invalid attachment uuid: %s", attachmentID)
}
canonicalID := parsedID.String()
if _, exists := seen[canonicalID]; exists {
return fmt.Errorf("duplicate attachment_id: %s", attachmentID)
}
seen[canonicalID] = struct{}{}
}
if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 {
return fmt.Errorf("message must contain non-blank text or at least one attachment")
}
return nil
}
// ValidateMessageText remains for callers that create text-only messages.
func ValidateMessageText(text string) error {
return ValidateMessageInput(text, nil)
}
+229
View File
@@ -0,0 +1,229 @@
package reconcile
import (
"context"
"fmt"
"sort"
"time"
"mtusi2026lab4/internal/adapter/objectstore"
"mtusi2026lab4/internal/domain"
)
const (
IssueOrphanObject = "orphan_object"
IssueMissingObject = "missing_object"
IssueStalePending = "stale_pending"
IssueFailedObject = "failed_object"
IssueReferencedMissing = "referenced_missing_object"
)
type AttachmentRecord struct {
ID string
ObjectKey string
Status domain.AttachmentStatus
UpdatedAt time.Time
Referenced bool
}
type MetadataStore interface {
ListAttachmentInventory(ctx context.Context) ([]AttachmentRecord, error)
MarkAttachmentFailedIfUnreferenced(
ctx context.Context,
attachmentID string,
expectedStatus domain.AttachmentStatus,
updatedBefore time.Time,
errorCode string,
updatedAt time.Time,
) (bool, error)
}
type ObjectInventory interface {
List(ctx context.Context, prefix string) ([]objectstore.StoredObject, error)
Delete(ctx context.Context, key string) error
}
type Issue struct {
Kind string `json:"kind"`
AttachmentID string `json:"attachment_id,omitempty"`
ObjectKey string `json:"object_key"`
Protected bool `json:"protected,omitempty"`
Action string `json:"action"`
}
type Result struct {
DryRun bool `json:"dry_run"`
GracePeriod time.Duration `json:"-"`
GracePeriodSeconds int64 `json:"grace_period_seconds"`
ScannedAttachments int `json:"scanned_attachments"`
ScannedObjects int `json:"scanned_objects"`
Issues []Issue `json:"issues"`
DeletedObjects int `json:"deleted_objects"`
FailedAttachments int `json:"failed_attachments"`
Duration time.Duration `json:"-"`
DurationMillis int64 `json:"duration_ms"`
}
type Reconciler struct {
metadata MetadataStore
objects ObjectInventory
now func() time.Time
}
func New(metadata MetadataStore, objects ObjectInventory) (*Reconciler, error) {
if metadata == nil || objects == nil {
return nil, fmt.Errorf("metadata and object stores are required")
}
return &Reconciler{
metadata: metadata,
objects: objects,
now: func() time.Time { return time.Now().UTC() },
}, nil
}
func (r *Reconciler) Run(
ctx context.Context,
gracePeriod time.Duration,
apply bool,
) (Result, error) {
if gracePeriod <= 0 {
return Result{}, fmt.Errorf("grace period must be positive")
}
startedAt := r.now().UTC()
cutoff := startedAt.Add(-gracePeriod)
result := Result{
DryRun: !apply,
GracePeriod: gracePeriod,
GracePeriodSeconds: int64(gracePeriod / time.Second),
Issues: make([]Issue, 0),
}
attachments, err := r.metadata.ListAttachmentInventory(ctx)
if err != nil {
return Result{}, fmt.Errorf("list attachment metadata: %w", err)
}
objects, err := r.objects.List(ctx, "attachments/")
if err != nil {
return Result{}, fmt.Errorf("list attachment objects: %w", err)
}
result.ScannedAttachments = len(attachments)
result.ScannedObjects = len(objects)
attachmentsByKey := make(map[string]AttachmentRecord, len(attachments))
for _, attachment := range attachments {
attachmentsByKey[attachment.ObjectKey] = attachment
}
objectsByKey := make(map[string]objectstore.StoredObject, len(objects))
for _, object := range objects {
objectsByKey[object.Key] = object
if _, exists := attachmentsByKey[object.Key]; exists {
continue
}
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
continue
}
issue := Issue{Kind: IssueOrphanObject, ObjectKey: object.Key, Action: "would_delete"}
if apply {
if err = r.objects.Delete(ctx, object.Key); err != nil {
return result, fmt.Errorf("delete orphan object %q: %w", object.Key, err)
}
issue.Action = "deleted"
result.DeletedObjects++
}
result.Issues = append(result.Issues, issue)
}
for _, attachment := range attachments {
if attachment.UpdatedAt.After(cutoff) {
continue
}
_, objectExists := objectsByKey[attachment.ObjectKey]
switch {
case attachment.Status == domain.AttachmentStatusReady && !objectExists:
issue := Issue{
Kind: IssueMissingObject,
AttachmentID: attachment.ID,
ObjectKey: attachment.ObjectKey,
Protected: attachment.Referenced,
Action: "would_mark_failed",
}
if attachment.Referenced {
issue.Kind = IssueReferencedMissing
issue.Action = "manual_repair_required"
} else if apply {
changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced(
ctx, attachment.ID, attachment.Status, cutoff,
"reconciler_missing_object", startedAt,
)
if markError != nil {
return result, fmt.Errorf("mark missing attachment %q failed: %w", attachment.ID, markError)
}
if changed {
issue.Action = "marked_failed"
result.FailedAttachments++
} else {
issue.Action = "skipped_concurrent_change"
}
}
result.Issues = append(result.Issues, issue)
case attachment.Status == domain.AttachmentStatusPending:
issue := Issue{
Kind: IssueStalePending, AttachmentID: attachment.ID,
ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced,
Action: "would_mark_failed",
}
if attachment.Referenced {
issue.Action = "manual_repair_required"
} else if apply {
changed, markError := r.metadata.MarkAttachmentFailedIfUnreferenced(
ctx, attachment.ID, attachment.Status, cutoff,
"reconciler_stale_pending", startedAt,
)
if markError != nil {
return result, fmt.Errorf("mark stale attachment %q failed: %w", attachment.ID, markError)
}
if changed {
issue.Action = "marked_failed"
result.FailedAttachments++
if objectExists {
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
return result, fmt.Errorf("delete stale pending object %q: %w", attachment.ObjectKey, err)
}
result.DeletedObjects++
}
} else {
issue.Action = "skipped_concurrent_change"
}
}
result.Issues = append(result.Issues, issue)
case attachment.Status == domain.AttachmentStatusFailed && objectExists:
issue := Issue{
Kind: IssueFailedObject, AttachmentID: attachment.ID,
ObjectKey: attachment.ObjectKey, Protected: attachment.Referenced,
Action: "would_delete",
}
if attachment.Referenced {
issue.Action = "manual_repair_required"
} else if apply {
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil {
return result, fmt.Errorf("delete failed attachment object %q: %w", attachment.ObjectKey, err)
}
issue.Action = "deleted"
result.DeletedObjects++
}
result.Issues = append(result.Issues, issue)
}
}
sort.Slice(result.Issues, func(i, j int) bool {
if result.Issues[i].ObjectKey == result.Issues[j].ObjectKey {
return result.Issues[i].Kind < result.Issues[j].Kind
}
return result.Issues[i].ObjectKey < result.Issues[j].ObjectKey
})
result.Duration = r.now().UTC().Sub(startedAt)
result.DurationMillis = result.Duration.Milliseconds()
return result, nil
}
+209
View File
@@ -0,0 +1,209 @@
package reconcile
import (
"context"
"errors"
"testing"
"time"
"mtusi2026lab4/internal/adapter/objectstore"
"mtusi2026lab4/internal/domain"
)
type fakeMetadataStore struct {
records []AttachmentRecord
markError error
skipMark bool
}
func (store *fakeMetadataStore) ListAttachmentInventory(
context.Context,
) ([]AttachmentRecord, error) {
return append([]AttachmentRecord(nil), store.records...), nil
}
func (store *fakeMetadataStore) MarkAttachmentFailedIfUnreferenced(
_ context.Context,
attachmentID string,
expectedStatus domain.AttachmentStatus,
updatedBefore time.Time,
errorCode string,
updatedAt time.Time,
) (bool, error) {
if store.markError != nil {
return false, store.markError
}
if store.skipMark {
return false, nil
}
for index := range store.records {
record := &store.records[index]
if record.ID != attachmentID || record.Status != expectedStatus ||
record.UpdatedAt.After(updatedBefore) || record.Referenced {
continue
}
record.Status = domain.AttachmentStatusFailed
record.UpdatedAt = updatedAt
_ = errorCode
return true, nil
}
return false, nil
}
func TestReconcilerDoesNotDeleteAfterConcurrentMetadataChange(t *testing.T) {
old := time.Now().UTC().Add(-2 * time.Hour)
metadata := &fakeMetadataStore{
records: []AttachmentRecord{{
ID: "pending-race", ObjectKey: "attachments/pending-race",
Status: domain.AttachmentStatusPending, UpdatedAt: old,
}},
skipMark: true,
}
objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{
"attachments/pending-race": {Key: "attachments/pending-race", LastModified: old},
}}
reconciler, err := New(metadata, objects)
if err != nil {
t.Fatal(err)
}
result, err := reconciler.Run(context.Background(), time.Hour, true)
if err != nil {
t.Fatal(err)
}
if result.DeletedObjects != 0 || result.FailedAttachments != 0 {
t.Fatalf("concurrent metadata change must suppress deletion: %+v", result)
}
if _, exists := objects.objects["attachments/pending-race"]; !exists {
t.Fatal("reconciler deleted an object after losing the metadata race")
}
if len(result.Issues) != 1 || result.Issues[0].Action != "skipped_concurrent_change" {
t.Fatalf("unexpected race result: %+v", result.Issues)
}
}
type fakeObjectInventory struct {
objects map[string]objectstore.StoredObject
deleteError error
}
func (store *fakeObjectInventory) List(
context.Context,
string,
) ([]objectstore.StoredObject, error) {
objects := make([]objectstore.StoredObject, 0, len(store.objects))
for _, object := range store.objects {
objects = append(objects, object)
}
return objects, nil
}
func (store *fakeObjectInventory) Delete(_ context.Context, key string) error {
if store.deleteError != nil {
return store.deleteError
}
delete(store.objects, key)
return nil
}
func TestReconcilerDryRunApplyAndRepeat(t *testing.T) {
now := time.Now().UTC()
old := now.Add(-2 * time.Hour)
fresh := now.Add(-5 * time.Minute)
metadata := &fakeMetadataStore{records: []AttachmentRecord{
{ID: "ready-present", ObjectKey: "attachments/ready-present", Status: domain.AttachmentStatusReady, UpdatedAt: old},
{ID: "ready-missing", ObjectKey: "attachments/ready-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old},
{ID: "ready-used-missing", ObjectKey: "attachments/ready-used-missing", Status: domain.AttachmentStatusReady, UpdatedAt: old, Referenced: true},
{ID: "pending-old", ObjectKey: "attachments/pending-old", Status: domain.AttachmentStatusPending, UpdatedAt: old},
{ID: "failed-old", ObjectKey: "attachments/failed-old", Status: domain.AttachmentStatusFailed, UpdatedAt: old},
{ID: "pending-fresh", ObjectKey: "attachments/pending-fresh", Status: domain.AttachmentStatusPending, UpdatedAt: fresh},
}}
objects := &fakeObjectInventory{objects: map[string]objectstore.StoredObject{
"attachments/ready-present": {Key: "attachments/ready-present", LastModified: old},
"attachments/pending-old": {Key: "attachments/pending-old", LastModified: old},
"attachments/failed-old": {Key: "attachments/failed-old", LastModified: old},
"attachments/orphan-old": {Key: "attachments/orphan-old", LastModified: old},
"attachments/orphan-fresh": {Key: "attachments/orphan-fresh", LastModified: fresh},
}}
reconciler, err := New(metadata, objects)
if err != nil {
t.Fatal(err)
}
dryRun, err := reconciler.Run(context.Background(), time.Hour, false)
if err != nil {
t.Fatal(err)
}
if !dryRun.DryRun || len(dryRun.Issues) != 5 ||
dryRun.DeletedObjects != 0 || dryRun.FailedAttachments != 0 {
t.Fatalf("unexpected dry-run result: %+v", dryRun)
}
if len(objects.objects) != 5 {
t.Fatal("dry-run changed object storage")
}
for _, record := range metadata.records {
if record.ID == "ready-missing" && record.Status != domain.AttachmentStatusReady {
t.Fatal("dry-run changed metadata")
}
}
applied, err := reconciler.Run(context.Background(), time.Hour, true)
if err != nil {
t.Fatal(err)
}
if applied.DeletedObjects != 3 || applied.FailedAttachments != 2 {
t.Fatalf("unexpected apply result: %+v", applied)
}
if _, exists := objects.objects["attachments/orphan-old"]; exists {
t.Fatal("old orphan was not deleted")
}
if _, exists := objects.objects["attachments/orphan-fresh"]; !exists {
t.Fatal("fresh orphan must be protected by grace period")
}
if _, exists := objects.objects["attachments/ready-present"]; !exists {
t.Fatal("healthy object was deleted")
}
for _, issue := range applied.Issues {
if issue.AttachmentID == "ready-used-missing" &&
(!issue.Protected || issue.Action != "manual_repair_required") {
t.Fatalf("referenced attachment was not protected: %+v", issue)
}
}
repeated, err := reconciler.Run(context.Background(), time.Hour, true)
if err != nil {
t.Fatal(err)
}
if repeated.DeletedObjects != 0 || repeated.FailedAttachments != 0 {
t.Fatalf("repeat must be mutation-idempotent: %+v", repeated)
}
if len(repeated.Issues) != 1 || !repeated.Issues[0].Protected {
t.Fatalf("repeat should retain only protected manual issue: %+v", repeated)
}
}
func TestReconcilerReportsMutationFailures(t *testing.T) {
old := time.Now().UTC().Add(-2 * time.Hour)
metadata := &fakeMetadataStore{records: []AttachmentRecord{
{ID: "missing", ObjectKey: "attachments/missing", Status: domain.AttachmentStatusReady, UpdatedAt: old},
}}
metadata.markError = errors.New("injected database failure")
objects := &fakeObjectInventory{objects: make(map[string]objectstore.StoredObject)}
reconciler, err := New(metadata, objects)
if err != nil {
t.Fatal(err)
}
if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil {
t.Fatal("injected database mutation failure was hidden")
}
metadata.records = nil
metadata.markError = nil
objects.objects["attachments/orphan"] = objectstore.StoredObject{
Key: "attachments/orphan", LastModified: old,
}
objects.deleteError = errors.New("injected object-store failure")
if _, err = reconciler.Run(context.Background(), time.Hour, true); err == nil {
t.Fatal("injected object-store mutation failure was hidden")
}
}
+59
View File
@@ -0,0 +1,59 @@
package dto
import (
"time"
"mtusi2026lab4/internal/domain"
)
type AttachmentVariantResponse struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type AttachmentResponse struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status domain.AttachmentStatus `json:"status"`
Variants []AttachmentVariantResponse `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func AttachmentFromDomain(attachment domain.Attachment) AttachmentResponse {
variants := make([]AttachmentVariantResponse, 0, len(attachment.Variants))
for _, variant := range attachment.Variants {
variants = append(variants, AttachmentVariantResponse{
Name: variant.Name,
ContentType: variant.ContentType,
SizeBytes: variant.SizeBytes,
Width: variant.Width,
Height: variant.Height,
SHA256: variant.SHA256,
})
}
return AttachmentResponse{
ID: attachment.ID,
ChatID: attachment.ChatID,
OwnerID: attachment.OwnerID,
Filename: attachment.Filename,
ContentType: attachment.ContentType,
SizeBytes: attachment.SizeBytes,
SHA256: attachment.SHA256,
Status: attachment.Status,
Variants: variants,
ErrorCode: attachment.ErrorCode,
CreatedAt: attachment.CreatedAt,
UpdatedAt: attachment.UpdatedAt,
}
}
+14 -7
View File
@@ -7,15 +7,17 @@ import (
)
type CreateMessageRequest struct {
Text string `json:"text"`
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}
type MessageResponse struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
CreatedAt time.Time `json:"created_at"`
}
type MessagePageResponse struct {
@@ -24,9 +26,14 @@ type MessagePageResponse struct {
}
func MessageFromDomain(message domain.Message) MessageResponse {
attachmentIDs := append([]string(nil), message.AttachmentIDs...)
if attachmentIDs == nil {
attachmentIDs = make([]string, 0)
}
return MessageResponse{
ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID,
Text: message.Text, CreatedAt: message.CreatedAt,
Text: message.Text, AttachmentIDs: attachmentIDs, CreatedAt: message.CreatedAt,
}
}
@@ -44,6 +44,7 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
ActorID: currentUser.ID,
ChatID: chatID,
Text: request.Text,
AttachmentIDs: request.AttachmentIDs,
Method: http.MethodPost,
CanonicalPath: "/api/v1/chats/" + chatID + "/messages",
IdempotencyKey: idempotencyKey,
@@ -53,6 +54,11 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
"Idempotency-Key is already used for another request", requestID)
return
}
if errors.Is(err, application.ErrAttachmentNotReady) {
response.WriteError(w, http.StatusConflict, response.CodeConflict,
"attachment is not ready", requestID)
return
}
if errors.Is(err, application.ErrInvalidInput) {
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": "), requestID)
@@ -0,0 +1,94 @@
package handlers
import (
"errors"
"io"
"mime"
"net/http"
"strconv"
"mtusi2026lab4/internal/application"
"mtusi2026lab4/internal/domain"
"mtusi2026lab4/internal/transport/http/dto"
"mtusi2026lab4/internal/transport/http/middleware"
"mtusi2026lab4/internal/transport/http/response"
"github.com/google/uuid"
)
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
requestID := middleware.GetRequestID(r.Context())
currentUser := middleware.GetCurrentUser(r.Context())
attachmentID, ok := parseAttachmentID(w, r, requestID)
if !ok {
return
}
attachment, err := h.service.GetAttachment(r.Context(), attachmentID, currentUser.ID)
if errors.Is(err, domain.ErrNotFound) {
response.WriteError(w, http.StatusNotFound, response.CodeNotFound,
"attachment not found", requestID)
return
}
if err != nil {
response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable,
"service is temporarily unavailable", requestID)
return
}
response.WriteJSON(w, http.StatusOK, dto.AttachmentFromDomain(attachment), requestID)
}
func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
requestID := middleware.GetRequestID(r.Context())
currentUser := middleware.GetCurrentUser(r.Context())
attachmentID, ok := parseAttachmentID(w, r, requestID)
if !ok {
return
}
download, err := h.service.DownloadAttachment(
r.Context(), attachmentID, currentUser.ID, r.URL.Query().Get("variant"),
)
switch {
case errors.Is(err, domain.ErrNotFound):
response.WriteError(w, http.StatusNotFound, response.CodeNotFound,
"attachment or variant not found", requestID)
return
case errors.Is(err, application.ErrAttachmentNotReady):
response.WriteError(w, http.StatusConflict, response.CodeConflict,
"attachment is not ready", requestID)
return
case errors.Is(err, application.ErrStorageUnavailable):
response.WriteError(w, http.StatusServiceUnavailable, response.CodeUnavailable,
"object storage is temporarily unavailable", requestID)
return
case err != nil:
response.WriteError(w, http.StatusInternalServerError, response.CodeInternalError,
"unexpected internal error", requestID)
return
}
defer download.Body.Close()
w.Header().Set("Content-Type", download.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(download.SizeBytes, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Content-Disposition", mime.FormatMediaType(
"attachment", map[string]string{"filename": download.Filename},
))
w.WriteHeader(http.StatusOK)
_, _ = io.Copy(w, download.Body)
}
func parseAttachmentID(
w http.ResponseWriter,
r *http.Request,
requestID string,
) (string, bool) {
parsedID, err := uuid.Parse(r.PathValue("attachment_id"))
if err != nil {
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
"invalid attachment_id uuid", requestID)
return "", false
}
return parsedID.String(), true
}
@@ -0,0 +1,97 @@
package handlers
import (
"errors"
"mime"
"net/http"
"strings"
"mtusi2026lab4/internal/application"
"mtusi2026lab4/internal/domain"
"mtusi2026lab4/internal/transport/http/dto"
"mtusi2026lab4/internal/transport/http/middleware"
"mtusi2026lab4/internal/transport/http/response"
"github.com/google/uuid"
)
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
requestID := middleware.GetRequestID(r.Context())
currentUser := middleware.GetCurrentUser(r.Context())
parsedChatID, err := uuid.Parse(r.PathValue("chat_id"))
if err != nil {
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
"invalid chat_id uuid", requestID)
return
}
filename := r.URL.Query().Get("filename")
if filename == "" {
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
"filename query parameter is required", requestID)
return
}
contentType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || len(parameters) != 0 ||
(contentType != "image/png" && contentType != "image/jpeg") {
response.WriteError(w, http.StatusUnsupportedMediaType,
response.CodeUnsupportedMediaType,
"Content-Type must be image/png or image/jpeg", requestID)
return
}
if r.ContentLength > domain.MaxAttachmentSizeBytes {
response.WriteError(w, http.StatusRequestEntityTooLarge,
response.CodePayloadTooLarge,
"attachment exceeds 10 MiB", requestID)
return
}
if r.Body == nil {
response.WriteError(w, http.StatusBadRequest, response.CodeInvalidRequest,
"attachment body is required", requestID)
return
}
defer r.Body.Close()
attachment, err := h.service.UploadAttachment(
r.Context(),
application.UploadAttachmentCommand{
ActorID: currentUser.ID,
ChatID: parsedChatID.String(),
Filename: filename,
DeclaredContentType: contentType,
Body: r.Body,
},
)
switch {
case errors.Is(err, application.ErrPayloadTooLarge):
response.WriteError(w, http.StatusRequestEntityTooLarge,
response.CodePayloadTooLarge,
"attachment exceeds 10 MiB or 40 million pixels", requestID)
return
case errors.Is(err, application.ErrUnsupportedMediaType), errors.Is(err, application.ErrInvalidImage):
response.WriteError(w, http.StatusUnsupportedMediaType,
response.CodeUnsupportedMediaType,
"body is not a valid PNG/JPEG matching the declared Content-Type", requestID)
return
case errors.Is(err, application.ErrInvalidInput):
message := strings.TrimPrefix(err.Error(), application.ErrInvalidInput.Error()+": ")
response.WriteError(w, http.StatusBadRequest,
response.CodeInvalidRequest, message, requestID)
return
case errors.Is(err, domain.ErrNotFound):
response.WriteError(w, http.StatusNotFound,
response.CodeNotFound, "chat not found", requestID)
return
case errors.Is(err, application.ErrStorageUnavailable):
response.WriteError(w, http.StatusServiceUnavailable,
response.CodeUnavailable, "object storage is temporarily unavailable", requestID)
return
case err != nil:
response.WriteError(w, http.StatusServiceUnavailable,
response.CodeUnavailable, "service is temporarily unavailable", requestID)
return
}
response.WriteJSON(w, http.StatusCreated, dto.AttachmentFromDomain(attachment), requestID)
}
+36
View File
@@ -0,0 +1,36 @@
BEGIN;
-- Downgrading would destroy the only content of attachment-only messages.
-- Refuse instead of silently losing valid lab4 data.
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM messages
WHERE btrim(text) = ''
) THEN
RAISE EXCEPTION
'cannot downgrade while attachment-only messages exist';
END IF;
END;
$$;
DROP TRIGGER message_attachments_payload_check ON message_attachments;
DROP TRIGGER messages_payload_check ON messages;
DROP FUNCTION enforce_message_payload_from_link();
DROP FUNCTION enforce_message_payload_from_message();
DROP TABLE message_attachments;
DROP TABLE attachments;
ALTER TABLE messages
DROP CONSTRAINT messages_id_chat_unique,
DROP CONSTRAINT messages_text_length;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length
CHECK (char_length(text) BETWEEN 1 AND 4000),
ADD CONSTRAINT messages_text_not_blank
CHECK (btrim(text) <> '');
COMMIT;
+191
View File
@@ -0,0 +1,191 @@
BEGIN;
-- Text may be empty only when the same transaction links at least one
-- attachment. A deferred constraint trigger below enforces that invariant.
ALTER TABLE messages
DROP CONSTRAINT messages_text_length,
DROP CONSTRAINT messages_text_not_blank;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length
CHECK (char_length(text) BETWEEN 0 AND 4000),
ADD CONSTRAINT messages_id_chat_unique
UNIQUE (id, chat_id);
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
owner_id UUID NOT NULL,
object_key TEXT NOT NULL,
filename TEXT NOT NULL,
content_type VARCHAR(32) NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 BYTEA NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
status VARCHAR(16) NOT NULL,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT attachments_id_chat_unique
UNIQUE (id, chat_id),
CONSTRAINT attachments_object_key_unique
UNIQUE (object_key),
CONSTRAINT attachments_owner_membership_foreign_key
FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id)
ON DELETE RESTRICT,
CONSTRAINT attachments_object_key_not_blank
CHECK (btrim(object_key) <> ''),
CONSTRAINT attachments_filename_length
CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_filename_safe
CHECK (
btrim(filename) <> ''
AND filename NOT IN ('.', '..')
AND position('/' in filename) = 0
AND position(chr(92) in filename) = 0
AND filename !~ '[[:cntrl:]]'
),
CONSTRAINT attachments_content_type_supported
CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_range
CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_length
CHECK (octet_length(sha256) = 32),
CONSTRAINT attachments_dimensions_positive
CHECK (width > 0 AND height > 0),
CONSTRAINT attachments_pixel_limit
CHECK (width::BIGINT * height::BIGINT <= 40000000),
CONSTRAINT attachments_status_supported
CHECK (status IN ('pending', 'ready', 'failed')),
CONSTRAINT attachments_status_error_consistency
CHECK (
(status = 'failed' AND error_code IS NOT NULL AND btrim(error_code) <> '')
OR
(status IN ('pending', 'ready') AND error_code IS NULL)
),
CONSTRAINT attachments_updated_after_creation
CHECK (updated_at >= created_at)
);
CREATE INDEX attachments_chat_created_at_id_idx
ON attachments (chat_id, created_at, id);
CREATE INDEX attachments_status_updated_at_idx
ON attachments (status, updated_at, id);
CREATE TABLE message_attachments (
message_id UUID NOT NULL,
chat_id UUID NOT NULL,
attachment_id UUID NOT NULL,
position SMALLINT NOT NULL,
CONSTRAINT message_attachments_primary_key
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_unique
UNIQUE (message_id, position),
CONSTRAINT message_attachments_position_range
CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_message_foreign_key
FOREIGN KEY (message_id, chat_id)
REFERENCES messages (id, chat_id)
ON DELETE CASCADE,
CONSTRAINT message_attachments_attachment_foreign_key
FOREIGN KEY (attachment_id, chat_id)
REFERENCES attachments (id, chat_id)
ON DELETE RESTRICT
);
CREATE INDEX message_attachments_attachment_id_idx
ON message_attachments (attachment_id, message_id);
CREATE FUNCTION enforce_message_payload_from_message()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
IF EXISTS (
SELECT 1
FROM messages AS message
WHERE message.id = NEW.id
AND btrim(message.text) = ''
AND NOT EXISTS (
SELECT 1
FROM message_attachments AS link
WHERE link.message_id = message.id
)
) THEN
RAISE EXCEPTION USING
ERRCODE = '23514',
CONSTRAINT = 'messages_content_required',
MESSAGE = 'message must contain non-blank text or an attachment';
END IF;
RETURN NULL;
END;
$$;
CREATE FUNCTION enforce_message_payload_from_link()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
affected_message_id UUID;
BEGIN
IF TG_OP = 'DELETE' THEN
affected_message_id := OLD.message_id;
ELSE
affected_message_id := NEW.message_id;
END IF;
IF EXISTS (
SELECT 1
FROM messages AS message
WHERE message.id = affected_message_id
AND btrim(message.text) = ''
AND NOT EXISTS (
SELECT 1
FROM message_attachments AS link
WHERE link.message_id = message.id
)
) THEN
RAISE EXCEPTION USING
ERRCODE = '23514',
CONSTRAINT = 'messages_content_required',
MESSAGE = 'message must contain non-blank text or an attachment';
END IF;
RETURN NULL;
END;
$$;
CREATE CONSTRAINT TRIGGER messages_payload_check
AFTER INSERT OR UPDATE ON messages
DEFERRABLE INITIALLY DEFERRED
FOR EACH ROW
EXECUTE FUNCTION enforce_message_payload_from_message();
CREATE CONSTRAINT TRIGGER message_attachments_payload_check
AFTER INSERT OR UPDATE OR DELETE ON message_attachments
DEFERRABLE INITIALLY DEFERRED
FOR EACH ROW
EXECUTE FUNCTION enforce_message_payload_from_link();
COMMIT;
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
set -eu
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
secrets_dir="$project_root/secrets"
command -v openssl >/dev/null 2>&1 || {
printf '%s\n' "openssl is required to generate local secrets" >&2
exit 1
}
umask 077
mkdir -p "$secrets_dir"
chmod 700 "$secrets_dir"
write_if_missing() {
destination=$1
value=$2
if [ ! -e "$destination" ]; then
printf '%s' "$value" > "$destination"
fi
chmod 600 "$destination"
}
random_hex() {
openssl rand -hex "$1"
}
write_if_missing "$secrets_dir/postgres_password.txt" "$(random_hex 24)"
postgres_password=$(cat "$secrets_dir/postgres_password.txt")
write_if_missing "$secrets_dir/database_url.txt" \
"postgres://messenger:${postgres_password}@postgres:5432/messenger?sslmode=disable"
write_if_missing "$secrets_dir/cursor_signing_key.txt" "$(random_hex 32)"
write_if_missing "$secrets_dir/minio_root_user.txt" "lab4-root"
write_if_missing "$secrets_dir/minio_root_password.txt" "$(random_hex 24)"
write_if_missing "$secrets_dir/s3_access_key_id.txt" "lab4-api"
write_if_missing "$secrets_dir/s3_secret_access_key.txt" "$(random_hex 24)"
printf '%s\n' "Local secret files are present with mode 600; existing values were not overwritten."
+12 -5
View File
@@ -6,13 +6,20 @@ cd "$project_root"
env_file=${ENV_FILE:-.env}
compose() {
docker compose --env-file "$env_file" "$@"
if [ -f "$env_file" ]; then
docker compose --env-file "$env_file" "$@"
else
docker compose "$@"
fi
}
# PostgreSQL хранит все обязательные данные и сессии в named volume.
# Redis содержит только восстанавливаемый кеш и намеренно не имеет volume.
compose restart postgres redis
compose up -d --wait postgres redis
# PostgreSQL хранит метаданные и сессии, MinIO — исходные байты вложений;
# оба используют named volumes. Redis содержит только восстанавливаемый кеш.
compose restart postgres redis minio
compose up -d --wait postgres redis minio
# Повторный bootstrap приватного bucket/policy/user обязан быть идемпотентным.
compose run --rm minio-bootstrap >/dev/null
# up восстанавливает желаемое число реплик, включая ранее вручную остановленную.
compose up -d --no-deps api
+9 -1
View File
@@ -9,7 +9,15 @@ cd "$project_root"
env_file=${ENV_FILE:-.env}
matched_container=
for container_id in $(docker compose --env-file "$env_file" ps -a -q api); do
compose() {
if [ -f "$env_file" ]; then
docker compose --env-file "$env_file" "$@"
else
docker compose "$@"
fi
}
for container_id in $(compose ps -a -q api); do
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
if [ -n "$matched_container" ]; then
+6 -2
View File
@@ -8,7 +8,11 @@ import (
const testCursorSigningKey = "student-tests-cursor-signing-key-at-least-32-bytes"
func newTestService(repository application.Repository, sessionTTL time.Duration) *application.Service {
func newTestService(
repository application.Repository,
sessionTTL time.Duration,
options ...application.ServiceOption,
) *application.Service {
if sessionTTL <= 0 {
sessionTTL = time.Hour
}
@@ -16,7 +20,7 @@ func newTestService(repository application.Repository, sessionTTL time.Duration)
if err != nil {
panic(err)
}
service, err := application.NewService(repository, codec, sessionTTL)
service, err := application.NewService(repository, codec, sessionTTL, options...)
if err != nil {
panic(err)
}
+256
View File
@@ -0,0 +1,256 @@
package student_tests
import (
"bytes"
"context"
"encoding/json"
"image"
"image/color"
"image/png"
"io"
"net/http"
"net/http/httptest"
"net/url"
"sync"
"testing"
"time"
store "mtusi2026lab4/internal/adapter/memory"
"mtusi2026lab4/internal/application"
"mtusi2026lab4/internal/domain"
"mtusi2026lab4/internal/transport/http/dto"
"mtusi2026lab4/internal/transport/http/handlers"
"mtusi2026lab4/internal/transport/http/middleware"
)
type httpFakeObjectStore struct {
mu sync.Mutex
objects map[string][]byte
}
func (store *httpFakeObjectStore) Put(_ context.Context, object application.ObjectPut) error {
payload, err := io.ReadAll(object.Body)
if err != nil {
return err
}
store.mu.Lock()
store.objects[object.Key] = payload
store.mu.Unlock()
return nil
}
func (store *httpFakeObjectStore) Get(_ context.Context, key string) (application.ObjectContent, error) {
store.mu.Lock()
defer store.mu.Unlock()
payload, exists := store.objects[key]
if !exists {
return application.ObjectContent{}, domain.ErrNotFound
}
copyOfPayload := append([]byte(nil), payload...)
return application.ObjectContent{
Body: io.NopCloser(bytes.NewReader(copyOfPayload)),
SizeBytes: int64(len(copyOfPayload)),
}, nil
}
func (store *httpFakeObjectStore) Delete(_ context.Context, key string) error {
store.mu.Lock()
delete(store.objects, key)
store.mu.Unlock()
return nil
}
func setupAttachmentUploadServer(
t *testing.T,
) (*httptest.Server, *store.MemoryStore, *httpFakeObjectStore) {
t.Helper()
memoryStore := store.NewMemoryStore()
repository := store.NewMemoryRepository(memoryStore)
objectStore := &httpFakeObjectStore{objects: make(map[string][]byte)}
service := newTestService(
repository,
time.Hour,
application.WithObjectStore(objectStore),
)
handler := handlers.NewHandler(service)
mux := http.NewServeMux()
mux.HandleFunc(
"POST /api/v1/chats/{chat_id}/attachments",
requireLegacyAuth(memoryStore, handler.UploadAttachment),
)
mux.HandleFunc(
"GET /api/v1/attachments/{attachment_id}",
requireLegacyAuth(memoryStore, handler.GetAttachment),
)
mux.HandleFunc(
"GET /api/v1/attachments/{attachment_id}/content",
requireLegacyAuth(memoryStore, handler.DownloadAttachment),
)
var root http.Handler = mux
root = middleware.Recovery(root)
root = middleware.RequestID(root)
return httptest.NewServer(root), memoryStore, objectStore
}
func pngPayload(t *testing.T) []byte {
t.Helper()
var payload bytes.Buffer
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
img.Set(0, 0, color.RGBA{R: 255, A: 255})
if err := png.Encode(&payload, img); err != nil {
t.Fatal(err)
}
return payload.Bytes()
}
func uploadRequest(
t *testing.T,
baseURL string,
chatID string,
userID string,
filename string,
contentType string,
payload []byte,
) *http.Response {
t.Helper()
requestURL := baseURL + "/api/v1/chats/" + chatID +
"/attachments?filename=" + url.QueryEscape(filename)
request, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(payload))
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", contentType)
request.Header.Set("X-User-Id", userID)
response, err := http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
return response
}
func attachmentGetRequest(t *testing.T, requestURL string, userID string) *http.Response {
t.Helper()
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
t.Fatal(err)
}
request.Header.Set("X-User-Id", userID)
response, err := http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
return response
}
func TestUploadAttachmentHTTPContract(t *testing.T) {
server, memoryStore, objectStore := setupAttachmentUploadServer(t)
defer server.Close()
user, err := memoryStore.CreateUser("attachment_user", "Attachment User")
if err != nil {
t.Fatal(err)
}
outsider, err := memoryStore.CreateUser("attachment_outsider", "Attachment Outsider")
if err != nil {
t.Fatal(err)
}
chat, err := memoryStore.CreateChat("Images", []string{user.ID})
if err != nil {
t.Fatal(err)
}
payload := pngPayload(t)
response := uploadRequest(t, server.URL, chat.ID, user.ID, "кот.png", "image/png", payload)
defer response.Body.Close()
if response.StatusCode != http.StatusCreated {
body, _ := io.ReadAll(response.Body)
t.Fatalf("status = %d, want 201: %s", response.StatusCode, body)
}
var attachment dto.AttachmentResponse
if err = json.NewDecoder(response.Body).Decode(&attachment); err != nil {
t.Fatal(err)
}
if attachment.Status != domain.AttachmentStatusReady || attachment.Filename != "кот.png" {
t.Fatalf("unexpected response: %+v", attachment)
}
if attachment.Variants == nil || len(attachment.Variants) != 0 || attachment.ErrorCode != nil {
t.Fatalf("contract arrays/nulls differ: %+v", attachment)
}
objectStore.mu.Lock()
storedObjects := len(objectStore.objects)
objectStore.mu.Unlock()
if storedObjects != 1 {
t.Fatalf("stored objects = %d, want 1", storedObjects)
}
metadataResponse := attachmentGetRequest(t, server.URL+"/api/v1/attachments/"+attachment.ID, user.ID)
defer metadataResponse.Body.Close()
if metadataResponse.StatusCode != http.StatusOK {
t.Fatalf("metadata status = %d, want 200", metadataResponse.StatusCode)
}
contentResponse := attachmentGetRequest(t,
server.URL+"/api/v1/attachments/"+attachment.ID+"/content", user.ID)
defer contentResponse.Body.Close()
downloaded, err := io.ReadAll(contentResponse.Body)
if err != nil {
t.Fatal(err)
}
if contentResponse.StatusCode != http.StatusOK || !bytes.Equal(downloaded, payload) {
t.Fatalf("download status/content differ: status=%d", contentResponse.StatusCode)
}
if contentResponse.Header.Get("Content-Type") != "image/png" {
t.Fatalf("download Content-Type = %q", contentResponse.Header.Get("Content-Type"))
}
for _, suffix := range []string{"", "/content"} {
outsiderResponse := attachmentGetRequest(t,
server.URL+"/api/v1/attachments/"+attachment.ID+suffix, outsider.ID)
_ = outsiderResponse.Body.Close()
if outsiderResponse.StatusCode != http.StatusNotFound {
t.Fatalf("outsider %q status = %d, want 404", suffix, outsiderResponse.StatusCode)
}
}
missingVariant := attachmentGetRequest(t,
server.URL+"/api/v1/attachments/"+attachment.ID+"/content?variant=thumbnail", user.ID)
_ = missingVariant.Body.Close()
if missingVariant.StatusCode != http.StatusNotFound {
t.Fatalf("missing variant status = %d, want 404", missingVariant.StatusCode)
}
}
func TestUploadAttachmentHTTPErrorMapping(t *testing.T) {
server, memoryStore, _ := setupAttachmentUploadServer(t)
defer server.Close()
user, _ := memoryStore.CreateUser("attachment_errors", "Attachment Errors")
chat, _ := memoryStore.CreateChat("Images", []string{user.ID})
validPNG := pngPayload(t)
testCases := []struct {
name string
chatID string
filename string
contentType string
payload []byte
wantStatus int
}{
{name: "unsafe filename", chatID: chat.ID, filename: "../image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusBadRequest},
{name: "unsupported content type", chatID: chat.ID, filename: "image.gif", contentType: "image/gif", payload: []byte("GIF89a"), wantStatus: http.StatusUnsupportedMediaType},
{name: "MIME substitution", chatID: chat.ID, filename: "image.jpg", contentType: "image/jpeg", payload: validPNG, wantStatus: http.StatusUnsupportedMediaType},
{name: "corrupt image", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: []byte("bad"), wantStatus: http.StatusUnsupportedMediaType},
{name: "too large", chatID: chat.ID, filename: "image.png", contentType: "image/png", payload: bytes.Repeat([]byte{0}, domain.MaxAttachmentSizeBytes+1), wantStatus: http.StatusRequestEntityTooLarge},
{name: "unknown chat", chatID: "11111111-1111-4111-8111-111111111111", filename: "image.png", contentType: "image/png", payload: validPNG, wantStatus: http.StatusNotFound},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
response := uploadRequest(t, server.URL, testCase.chatID, user.ID,
testCase.filename, testCase.contentType, testCase.payload)
defer response.Body.Close()
if response.StatusCode != testCase.wantStatus {
body, _ := io.ReadAll(response.Body)
t.Fatalf("status = %d, want %d: %s", response.StatusCode, testCase.wantStatus, body)
}
})
}
}
+40 -2
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"reflect"
"sync"
"testing"
@@ -133,7 +134,7 @@ func TestMessageIdempotencyReplayAndConflict(t *testing.T) {
first := decodeMessageResponse(t, firstPayload)
replay := decodeMessageResponse(t, replayPayload)
if first != replay {
if !reflect.DeepEqual(first, replay) {
t.Fatalf("replay changed message: first=%+v replay=%+v", first, replay)
}
@@ -256,7 +257,7 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) {
for message := range results {
if count == 0 {
expected = message
} else if message != expected {
} else if !reflect.DeepEqual(message, expected) {
t.Fatalf("concurrent replay returned different message: %+v != %+v", message, expected)
}
count++
@@ -273,3 +274,40 @@ func TestConcurrentMessageIdempotencyCreatesExactlyOneMessage(t *testing.T) {
t.Fatalf("expected exactly one stored message, got %d", len(messages))
}
}
func TestMessageIdempotencyTreatsMissingAndEmptyAttachmentsEqually(t *testing.T) {
_, serverURL, user, chatID := createMessageFixture(t)
status, firstPayload := sendMessageRequest(
t,
serverURL,
chatID,
user.ID,
"default-attachments",
`{"text":"same semantic request"}`,
)
if status != http.StatusCreated {
t.Fatalf("first request: expected 201, got %d: %s", status, firstPayload)
}
status, replayPayload := sendMessageRequest(
t,
serverURL,
chatID,
user.ID,
"default-attachments",
`{"attachment_ids":[],"text":"same semantic request"}`,
)
if status != http.StatusCreated {
t.Fatalf("replay: expected 201, got %d: %s", status, replayPayload)
}
first := decodeMessageResponse(t, firstPayload)
replay := decodeMessageResponse(t, replayPayload)
if !reflect.DeepEqual(first, replay) {
t.Fatalf("missing attachment_ids and [] must replay identically")
}
if first.AttachmentIDs == nil || len(first.AttachmentIDs) != 0 {
t.Fatalf("response must contain attachment_ids: [], got %#v", first.AttachmentIDs)
}
}
+133
View File
@@ -0,0 +1,133 @@
package student_tests
import (
"fmt"
"net/http"
"reflect"
"testing"
"mtusi2026lab4/internal/domain"
)
func TestMessageAttachmentContract(t *testing.T) {
memoryStore, serverURL, user, chatID := createMessageFixture(t)
readyID := "11111111-1111-4111-8111-111111111111"
if err := memoryStore.PutAttachment(domain.Attachment{
ID: readyID,
ChatID: chatID,
OwnerID: user.ID,
Status: domain.AttachmentStatusReady,
}); err != nil {
t.Fatalf("seed ready attachment: %v", err)
}
status, payload := sendMessageRequest(
t,
serverURL,
chatID,
user.ID,
"attachment-only",
fmt.Sprintf(`{"text":"","attachment_ids":[%q]}`, readyID),
)
if status != http.StatusCreated {
t.Fatalf("attachment-only message: expected 201, got %d: %s", status, payload)
}
message := decodeMessageResponse(t, payload)
if message.Text != "" {
t.Fatalf("attachment-only message changed text: %q", message.Text)
}
if !reflect.DeepEqual(message.AttachmentIDs, []string{readyID}) {
t.Fatalf("attachment_ids changed: %#v", message.AttachmentIDs)
}
items, _, err := memoryStore.GetMessages(chatID, user.ID, 10, nil)
if err != nil {
t.Fatalf("read stored messages: %v", err)
}
if len(items) != 1 || !reflect.DeepEqual(items[0].AttachmentIDs, []string{readyID}) {
t.Fatalf("stored attachment relation differs: %#v", items)
}
}
func TestMessageAttachmentValidationAndAccess(t *testing.T) {
memoryStore, serverURL, user, chatID := createMessageFixture(t)
otherChat, err := memoryStore.CreateChat("Other chat", []string{user.ID})
if err != nil {
t.Fatalf("create other chat: %v", err)
}
pendingID := "22222222-2222-4222-8222-222222222222"
if err = memoryStore.PutAttachment(domain.Attachment{
ID: pendingID,
ChatID: chatID,
OwnerID: user.ID,
Status: domain.AttachmentStatusPending,
}); err != nil {
t.Fatalf("seed pending attachment: %v", err)
}
foreignID := "33333333-3333-4333-8333-333333333333"
if err = memoryStore.PutAttachment(domain.Attachment{
ID: foreignID,
ChatID: otherChat.ID,
OwnerID: user.ID,
Status: domain.AttachmentStatusReady,
}); err != nil {
t.Fatalf("seed foreign attachment: %v", err)
}
testCases := []struct {
name string
body string
wantStatus int
}{
{
name: "empty without attachments",
body: `{"text":"","attachment_ids":[]}`,
wantStatus: http.StatusBadRequest,
},
{
name: "invalid attachment uuid",
body: `{"text":"hello","attachment_ids":["not-a-uuid"]}`,
wantStatus: http.StatusBadRequest,
},
{
name: "duplicate attachment",
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q,%q]}`, pendingID, pendingID),
wantStatus: http.StatusBadRequest,
},
{
name: "unknown attachment",
body: `{"text":"hello","attachment_ids":["44444444-4444-4444-8444-444444444444"]}`,
wantStatus: http.StatusNotFound,
},
{
name: "attachment from another chat",
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, foreignID),
wantStatus: http.StatusNotFound,
},
{
name: "attachment not ready",
body: fmt.Sprintf(`{"text":"hello","attachment_ids":[%q]}`, pendingID),
wantStatus: http.StatusConflict,
},
}
for index, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
status, payload := sendMessageRequest(
t,
serverURL,
chatID,
user.ID,
fmt.Sprintf("attachment-validation-%d", index),
testCase.body,
)
if status != testCase.wantStatus {
t.Fatalf("expected %d, got %d: %s", testCase.wantStatus, status, payload)
}
})
}
}
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Black-box attachment validation and exact contract-boundary checks."""
import os
import struct
import zlib
from attachment_memory import cleanup, noisy_png, png_chunk
from client import Client, check, run
MAX_BYTES = 10 * 1024 * 1024
def exact_size_png(sample, target_size=MAX_BYTES):
check(sample.endswith(png_chunk(b"IEND", b"")), "sample PNG has no final IEND")
prefix, iend = sample[:-12], sample[-12:]
filler_size = target_size - len(prefix) - len(iend) - 12
check(filler_size >= 0, "sample PNG is larger than boundary")
result = prefix + png_chunk(b"ruSt", bytes(filler_size)) + iend
check(len(result) == target_size, "exact-size PNG construction failed")
return result
def exact_pixel_png(width=10_000, height=4_000):
compressor = zlib.compressobj(level=9)
row = bytes(1 + width * 3)
parts = []
for _ in range(height):
chunk = compressor.compress(row)
if chunk:
parts.append(chunk)
parts.append(compressor.flush())
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ png_chunk(b"IDAT", b"".join(parts))
+ png_chunk(b"IEND", b"")
)
def config_only_png(width, height):
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
)
def upload(client, actor, chat, filename, payload, expected, schema=None):
return client.call(
"POST",
"/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename),
expected,
actor=actor,
raw=payload,
headers={"Content-Type": "image/png"},
schema=schema,
)
def main():
client = Client(os.environ.get("BASE_URL"), None, timeout=30)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
sample = open(os.path.join(os.path.dirname(__file__), "fixtures", "sample.png"), "rb").read()
try:
boundary = exact_size_png(sample)
response = upload(client, alice, chat, "exact-10mib.png", boundary, 201, "Attachment")
attachment = response["json"]
check(attachment["size_bytes"] == MAX_BYTES, "10 MiB boundary size changed")
downloaded = client.call(
"GET", "/api/v1/attachments/%s/content" % attachment["id"],
200, actor=alice,
)["bytes"]
check(downloaded == boundary, "10 MiB original changed after round trip")
pixels = exact_pixel_png()
upload(client, alice, chat, "exact-40m-pixels.png", pixels, 201, "Attachment")
upload(client, alice, chat, "over-pixels.png", config_only_png(20_000, 10_000), 413)
upload(client, alice, chat, "truncated.png", sample[:33], 415)
corrupted = noisy_png()
check(len(corrupted) > 70_000, "corruption fixture is too small")
corrupted = bytearray(corrupted)
corrupted[70_000] ^= 0xff
upload(client, alice, chat, "corrupt-after-prefix.png", bytes(corrupted), 415)
upload(client, alice, chat, "%20%20%20", sample, 400)
upload(client, alice, chat, "c1%C2%85.png", sample, 400)
upload(client, alice, chat, "invalid-%FF.png", sample, 400)
print("PASS: exact byte/pixel boundaries, complete decode, and filename validation are correct.")
finally:
cleanup(chat["id"], users)
if __name__ == "__main__":
run(main)
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Concurrent real PNG uploads with Docker API-container memory sampling."""
import concurrent.futures
import json
import os
import struct
import subprocess
import threading
import time
import uuid
import zlib
from client import Client, check, run
def png_chunk(kind, payload):
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload))
def noisy_png(width=2048, height=1024):
rows = bytearray()
for _ in range(height):
rows.append(0)
rows.extend(os.urandom(width * 3))
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ png_chunk(b"IDAT", zlib.compress(bytes(rows), level=0))
+ png_chunk(b"IEND", b"")
)
def bytes_from_docker(value):
number, unit = value.split()[0], value.split()[0][-3:]
if unit in {"KiB", "MiB", "GiB"}:
number = value.split()[0][:-3]
else:
unit = "B"
factor = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}[unit]
return int(float(number) * factor)
def api_container_ids():
result = subprocess.run(
["docker", "compose", "ps", "-q", "api"],
check=True, text=True, capture_output=True,
)
return [line for line in result.stdout.splitlines() if line]
def sample_memory(container_ids, stop_event, peaks):
while not stop_event.is_set():
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}", *container_ids],
check=False, text=True, capture_output=True,
)
if result.returncode == 0:
for line in result.stdout.splitlines():
name, usage = line.split("\t", 1)
current = bytes_from_docker(usage.split(" / ", 1)[0])
peaks[name] = max(peaks.get(name, 0), current)
stop_event.wait(0.05)
def cleanup(chat_id, user_ids):
object_script = (
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'mc rm --recursive --force "local/${S3_BUCKET}/attachments/' + chat_id + '/" >/dev/null 2>&1 || true\n'
)
subprocess.run(
["docker", "compose", "run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", object_script],
check=True,
)
quoted_users = ",".join("'%s'" % value for value in user_ids)
sql = f"""
BEGIN;
DELETE FROM messages WHERE chat_id = '{chat_id}';
DELETE FROM attachments WHERE chat_id = '{chat_id}';
DELETE FROM sessions WHERE user_id IN ({quoted_users});
DELETE FROM chat_members WHERE chat_id = '{chat_id}';
DELETE FROM chats WHERE id = '{chat_id}';
DELETE FROM users WHERE id IN ({quoted_users});
COMMIT;
"""
subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger"],
input=sql, text=True, check=True,
)
def main():
concurrency = 12
base_url = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
client = Client(base_url, None, timeout=30)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
payload = noisy_png()
stop_event = threading.Event()
peaks = {}
sampler = threading.Thread(
target=sample_memory,
args=(api_container_ids(), stop_event, peaks),
daemon=True,
)
sampler.start()
started = time.monotonic()
def upload(index):
response = client.call(
"POST",
"/api/v1/chats/%s/attachments?filename=memory-%02d.png" % (chat["id"], index),
201,
actor=alice,
raw=payload,
headers={"Content-Type": "image/png", "X-Load-Id": uuid.uuid4().hex},
schema="Attachment",
)
return response["headers"]["X-Instance-Id"]
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
instances = list(pool.map(upload, range(concurrency)))
elapsed = time.monotonic() - started
stop_event.set()
sampler.join(timeout=10)
check(len(set(instances)) >= 2, "uploads did not exercise both API replicas")
check(len(peaks) >= 2, "memory was not sampled for both API replicas")
summary = {
"concurrency": concurrency,
"payload_bytes": len(payload),
"total_uploaded_bytes": len(payload) * concurrency,
"elapsed_seconds": round(elapsed, 3),
"successful_uploads": len(instances),
"api_instances_observed": len(set(instances)),
"peak_memory_bytes_by_container": peaks,
}
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
print("PASS: concurrent attachment uploads remained within configured API memory limits.")
finally:
stop_event.set()
sampler.join(timeout=10)
cleanup(chat["id"], users)
if __name__ == "__main__":
run(main)
+52
View File
@@ -0,0 +1,52 @@
#!/bin/sh
set -eu
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$project_root"
api_containers=$(docker compose ps -q api)
if [ -z "$api_containers" ]; then
printf "%s\n" "no running API containers found" >&2
exit 1
fi
for container_id in $api_containers; do
docker exec "$container_id" sh -ec '
test ! -e /run/secrets/minio_root_user
test ! -e /run/secrets/minio_root_password
! env | grep -Eq "^MINIO_ROOT_(USER|PASSWORD)="
'
done
docker compose run --rm --entrypoint /bin/sh minio-bootstrap -ec '
root_user="$(cat /run/secrets/minio_root_user)"
root_password="$(cat /run/secrets/minio_root_password)"
api_user="$(cat /run/secrets/s3_access_key_id)"
api_password="$(cat /run/secrets/s3_secret_access_key)"
probe_key="policy-probe/object.txt"
mc alias set admin http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null
mc alias set limited http://minio:9000 "$api_user" "$api_password" --api S3v4 >/dev/null
cleanup() {
mc rm --force "admin/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1 || true
mc rb --force "admin/forbidden-policy-probe" >/dev/null 2>&1 || true
}
trap cleanup EXIT INT TERM
printf private-probe | mc pipe "limited/${S3_BUCKET}/${probe_key}" >/dev/null
mc stat "limited/${S3_BUCKET}/${probe_key}" >/dev/null
mc cat "limited/${S3_BUCKET}/${probe_key}" >/dev/null
if mc mb "limited/forbidden-policy-probe" >/dev/null 2>&1; then
printf "%s\n" "limited API credentials unexpectedly created another bucket" >&2
exit 1
fi
if wget -qO- "http://minio:9000/${S3_BUCKET}/${probe_key}" >/dev/null 2>&1; then
printf "%s\n" "anonymous HTTP GET unexpectedly read a private object" >&2
exit 1
fi
mc rm --force "limited/${S3_BUCKET}/${probe_key}" >/dev/null
trap - EXIT INT TERM
printf "%s\n" "PASS: API has no MinIO root credentials; limited policy permits object IO only in the private bucket; anonymous GET is denied."
'
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Inject isolated DB/S3 inconsistencies and prove reconciler dry-run/apply/repeat."""
import json
import os
import subprocess
import time
USER_ID = "90000000-0000-4000-8000-000000000001"
CHAT_ID = "90000000-0000-4000-8000-000000000002"
MISSING_ID = "90000000-0000-4000-8000-000000000011"
PROTECTED_ID = "90000000-0000-4000-8000-000000000012"
PENDING_ID = "90000000-0000-4000-8000-000000000013"
MESSAGE_ID = "90000000-0000-4000-8000-000000000021"
PREFIX = "attachments/reconcile-integration"
ORPHAN_KEY = PREFIX + "/orphan"
PENDING_KEY = PREFIX + "/pending"
def run(command, *, input_text=None, capture=False):
result = subprocess.run(
command,
input=input_text,
text=True,
check=False,
capture_output=capture,
env=os.environ.copy(),
)
if result.returncode != 0:
details = result.stderr.strip() if capture else "see command output"
raise RuntimeError("command failed: %s: %s" % (command[0], details))
return result.stdout if capture else ""
def compose(*arguments, capture=False, input_text=None):
return run(
["docker", "compose", *arguments],
input_text=input_text,
capture=capture,
)
def database(sql):
compose(
"exec", "-T", "postgres",
"psql", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger",
input_text=sql,
)
def object_command(script):
compose(
"run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap",
"-ec", script,
)
def clean_objects():
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'mc rm --force "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null 2>&1 || true\n'
'mc rm --force "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null 2>&1 || true\n'
)
def clean_database():
database(f"""
BEGIN;
DELETE FROM messages WHERE id = '{MESSAGE_ID}';
DELETE FROM attachments WHERE id IN ('{MISSING_ID}', '{PROTECTED_ID}', '{PENDING_ID}');
DELETE FROM chat_members WHERE chat_id = '{CHAT_ID}';
DELETE FROM chats WHERE id = '{CHAT_ID}';
DELETE FROM users WHERE id = '{USER_ID}';
COMMIT;
""")
def inject_database():
database(f"""
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER_ID}', 'reconcile_fixture', 'Reconcile Fixture', 'not-used');
INSERT INTO chats (id, title) VALUES ('{CHAT_ID}', 'Reconcile Integration');
INSERT INTO chat_members (chat_id, user_id, position) VALUES ('{CHAT_ID}', '{USER_ID}', 0);
INSERT INTO attachments (
id, chat_id, owner_id, object_key, filename, content_type,
size_bytes, sha256, width, height, status, created_at, updated_at
) VALUES
('{MISSING_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/missing', 'missing.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'),
('{PROTECTED_ID}', '{CHAT_ID}', '{USER_ID}', '{PREFIX}/protected-missing', 'protected.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'ready', now() - interval '2 hours', now() - interval '2 hours'),
('{PENDING_ID}', '{CHAT_ID}', '{USER_ID}', '{PENDING_KEY}', 'pending.png', 'image/png',
7, decode(repeat('00', 32), 'hex'), 1, 1, 'pending', now() - interval '2 hours', now() - interval '2 hours');
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{USER_ID}', 'protected attachment reference');
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES ('{MESSAGE_ID}', '{CHAT_ID}', '{PROTECTED_ID}', 0);
COMMIT;
""")
def inject_objects():
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'printf orphan | mc pipe "local/${S3_BUCKET}/' + ORPHAN_KEY + '" >/dev/null\n'
'printf pending | mc pipe "local/${S3_BUCKET}/' + PENDING_KEY + '" >/dev/null\n'
)
def reconcile(apply=False):
command = [
"run", "--rm", "--entrypoint", "/app/reconciler", "api",
"--grace-period", "1s",
]
if apply:
command.append("--apply")
output = compose(*command, capture=True)
return json.loads(output)
def kinds(result):
return {issue["kind"] for issue in result["issues"]}
def main():
clean_objects()
clean_database()
try:
inject_database()
inject_objects()
time.sleep(2)
dry_run = reconcile(False)
expected = {
"orphan_object", "missing_object", "referenced_missing_object", "stale_pending",
}
if not expected.issubset(kinds(dry_run)):
raise RuntimeError("dry-run did not detect all injected inconsistencies")
if dry_run["deleted_objects"] or dry_run["failed_attachments"]:
raise RuntimeError("dry-run performed mutations")
applied = reconcile(True)
if applied["deleted_objects"] != 2 or applied["failed_attachments"] != 2:
raise RuntimeError("apply mutation counts differ: %s" % applied)
repeated = reconcile(True)
if repeated["deleted_objects"] or repeated["failed_attachments"]:
raise RuntimeError("repeated apply was not mutation-idempotent")
protected = [issue for issue in repeated["issues"] if issue.get("protected")]
if len(protected) != 1 or protected[0]["action"] != "manual_repair_required":
raise RuntimeError("referenced missing attachment was not protected")
print(json.dumps({
"dry_run_issues": len(dry_run["issues"]),
"apply_deleted_objects": applied["deleted_objects"],
"apply_failed_attachments": applied["failed_attachments"],
"repeat_mutations": repeated["deleted_objects"] + repeated["failed_attachments"],
"protected_manual_issues": len(protected),
"scanned_attachments": applied["scanned_attachments"],
"scanned_objects": applied["scanned_objects"],
"duration_ms": applied["duration_ms"],
}, ensure_ascii=False))
print("PASS: real PostgreSQL/MinIO reconciliation is dry-run safe, idempotent and reference-aware.")
finally:
clean_objects()
clean_database()
if __name__ == "__main__":
main()
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Measure real PostgreSQL/MinIO reconciler traversal at two inventory sizes."""
import json
import subprocess
import time
PREFIX = "attachments/reconcile-scale"
def compose(*arguments, capture=False):
result = subprocess.run(
["docker", "compose", *arguments],
check=False,
text=True,
capture_output=capture,
)
if result.returncode != 0:
raise RuntimeError(result.stderr.strip() if capture else "docker compose command failed")
return result.stdout if capture else ""
def object_command(script):
compose("run", "--rm", "--entrypoint", "/bin/sh", "minio-bootstrap", "-ec", script)
def clean():
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'mc rm --recursive --force "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null 2>&1 || true\n'
)
def create(count):
object_command(
'root_user="$(cat /run/secrets/minio_root_user)"\n'
'root_password="$(cat /run/secrets/minio_root_password)"\n'
'mc alias set local http://minio:9000 "$root_user" "$root_password" --api S3v4 >/dev/null\n'
'mkdir -p /tmp/reconcile-scale\n'
'index=0\n'
'while [ "$index" -lt ' + str(count) + ' ]; do\n'
' printf x > "/tmp/reconcile-scale/object-${index}"\n'
' index=$((index + 1))\n'
'done\n'
'mc cp --recursive /tmp/reconcile-scale/ "local/${S3_BUCKET}/' + PREFIX + '/" >/dev/null\n'
)
def dry_run():
started = time.monotonic()
output = compose(
"run", "--rm", "--entrypoint", "/app/reconciler", "api",
"--grace-period", "1s",
capture=True,
)
elapsed = time.monotonic() - started
result = json.loads(output)
return {
"scanned_attachments": result["scanned_attachments"],
"scanned_objects": result["scanned_objects"],
"issues": len(result["issues"]),
"reconciler_duration_ms": result["duration_ms"],
"container_start_and_run_ms": round(elapsed * 1000),
}
def main():
measurements = []
clean()
try:
for count in (100, 1000):
clean()
create(count)
time.sleep(2)
measurement = dry_run()
measurement["injected_orphan_objects"] = count
if measurement["issues"] < count:
raise RuntimeError("reconciler did not observe all scale fixtures")
measurements.append(measurement)
print(json.dumps({"measurements": measurements}, ensure_ascii=False))
print("PASS: reconciler traversal cost measured on real PostgreSQL/MinIO inventories.")
finally:
clean()
if __name__ == "__main__":
main()
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Verify contract responses and original durability during a real MinIO outage."""
import json
import os
import subprocess
import time
from attachment_memory import cleanup
from client import Client, Failure, check, run
def compose(*arguments, **kwargs):
return subprocess.run(["docker", "compose", *arguments], check=True, **kwargs)
def wait_for_status(client, path, expected, timeout=20):
deadline = time.monotonic() + timeout
last_error = None
while time.monotonic() < deadline:
try:
return client.call("GET", path, expected=expected)
except Failure as error:
last_error = error
time.sleep(0.25)
raise Failure("%s did not reach %d: %s" % (path, expected, last_error))
def restore_storage(client):
compose("up", "-d", "--wait", "minio")
compose("run", "--rm", "minio-bootstrap")
wait_for_status(client, "/health/ready", 200, timeout=30)
def main():
client = Client(os.environ.get("BASE_URL"), None, timeout=15)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
attachment, original = client.attachment(alice, chat)
storage_stopped = False
try:
compose("stop", "minio")
storage_stopped = True
readiness = wait_for_status(client, "/health/ready", 503)
check(readiness["elapsed"] < 5, "readiness exceeded its bounded timeout")
started = time.monotonic()
failure = client.call(
"POST",
"/api/v1/chats/%s/attachments?filename=outage.png" % chat["id"],
503,
actor=alice,
raw=original,
headers={"Content-Type": "image/png"},
)
elapsed = time.monotonic() - started
check(failure["json"]["error"]["code"] == "unavailable", "wrong S3 outage error code")
check(elapsed < 15, "upload outage response exceeded API/client timeout")
restore_storage(client)
storage_stopped = False
downloaded = client.call(
"GET", "/api/v1/attachments/%s/content" % attachment["id"],
200, actor=alice,
)["bytes"]
check(downloaded == original, "existing original changed across MinIO outage")
print(json.dumps({
"readiness_503_seconds": round(readiness["elapsed"], 3),
"upload_503_seconds": round(elapsed, 3),
"original_bytes_verified": len(original),
}, sort_keys=True))
print("PASS: MinIO outage returns bounded JSON 503 and preserves existing originals.")
finally:
if storage_stopped:
restore_storage(client)
cleanup(chat["id"], users)
if __name__ == "__main__":
run(main)