diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d96d7124 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# --- build stage --- +FROM golang:1.26-alpine AS build + +WORKDIR /src +RUN apk add --no-cache ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY src ./src +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger && \ + CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/outbox ./cmd/outbox && \ + CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + +# --- runtime stage --- +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates postgresql-client && \ + addgroup -S app && adduser -S -G app app +WORKDIR /app + +COPY --from=build /out/messenger /app/messenger +COPY --from=build /out/worker /app/worker +COPY --from=build /out/outbox /app/outbox +COPY migrations /app/migrations +COPY scripts/migrate.sh /app/migrate.sh +RUN chmod +x /app/migrate.sh + +USER app +EXPOSE 8080 + +ENTRYPOINT ["/app/messenger"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d171..943cf6ca 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover my_tests my_tests_quick my_tests_slow help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,24 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# Быстрые (<1 мин): статусы, лимиты, варианты, payload, DLQ, staging. +my_tests_quick: + python3 student_tests/states.py + python3 student_tests/limits.py + python3 student_tests/variants.py + python3 student_tests/broker-payload.py + python3 student_tests/staging-cleanup.py + python3 student_tests/dlq.py + python3 student_tests/redelivery.py + +# Долгие (>2 мин): retry/backoff/failed/replay/outbox/lease/crash. +my_tests_slow: + python3 student_tests/outbox.py + python3 student_tests/failed.py + python3 student_tests/retry.py + python3 student_tests/replay.py + python3 student_tests/lease.py + python3 student_tests/crash-before-ack.py + +my_tests: my_tests_quick my_tests_slow \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..e32171da 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,184 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Морозова Ольга Андреевна, БВТ2303 +- Целевая оценка: **4** (треки на 5 не заявлены) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники: [README.md](README.md), [COURSE.md](COURSE.md), [contracts/openapi.json](contracts/openapi.json), документация Go, MinIO (mc), PostgreSQL, Redis Streams, Nginx, Docker. +- Библиотеки: Go 1.26 (stdlib), github.com/google/uuid v1.6.0, github.com/jackc/pgx/v5 v5.5.5, github.com/redis/go-redis/v9 v9.5.1, golang.org/x/crypto v0.23.0, github.com/minio/minio-go/v7 v7.0.72, golang.org/x/image v0.48.0. Тесты — Python 3 (stdlib). +- Помощь ИИ: Deepseek — написание и проверка кода. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Команды выполнялись в WSL2 (Ubuntu) на Windows 11. Требования: Docker 24+ (Compose v2), Python 3.9+ (для тестов), `make`. + +Версии (фиксированные теги образов): + +- `golang:1.26-alpine` — build stage +- `alpine:3.20` — runtime stage +- `postgres:16.3-alpine` — база данных +- `redis:7.2.5-alpine` — кэш сессий и брокер (Redis Streams) +- `nginx:1.27.0-alpine` — балансировщик +- `pgsty/silo:RELEASE.2026-10-26T00-00-00Z` — S3-совместимое хранилище (MinIO-compatible) + +### Подготовка + +```sh +git clone https://gitea.privcloud.tech/MiniLynx/mtusi2026lab5-fork +cd mtusi2026lab5-fork + +# 1. Секреты (не входят в Git) +mkdir -p secrets +printf 'local-dev-postgres-password\n' > secrets/postgres_password.txt +printf 'postgres://messenger:local-dev-postgres-password@postgres:5432/messenger?sslmode=disable\n' > secrets/database_url.txt +printf 'redis://redis:6379/0\n' > secrets/cache_url.txt +printf 'minioadmin\n' > secrets/minio_root_user.txt +printf 'minioadmin-dev-password\n' > secrets/minio_root_password.txt +printf 'messenger-api\n' > secrets/s3_access_key_id.txt +printf 'messenger-api-dev-secret\n' > secrets/s3_secret_access_key.txt +chmod 600 secrets/*.txt + +# 2. Локальный .env +cp .env.example .env +# ВАЖНО: В .env нужно исправить S3_ENDPOINT на s3:9000 (без http://) +sed -i 's|S3_ENDPOINT=.*|S3_ENDPOINT=s3:9000|' .env + +# 3. Python для проверок +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements-dev.txt + +# 4. Скрипты +chmod +x scripts/migrate.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh +``` + +### Сборка и запуск + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться /health/ready +docker compose ps +``` + +Для запуска с другим количеством реплик добавить `API_REPLICAS=$n` перед `docker compose --env-file .env up -d --build`, где `$n` - желаемое количество реплик. + +### Тесты + +```sh +make check +make validate PYTHON=.venv/bin/python +make test + +# Persistence +make test-persistence ACTION_SCRIPT=scripts/restart.sh + +# Failover +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +./scripts/restore-api.sh + +# Собственные тесты (pipeline, сбои, лимиты) +make my_tests +``` + +## Остановка (с сохранением volumes) + +```sh +docker compose --env-file .env down --remove-orphans +``` + +### Намеренное удаление учебных данных + +```sh +docker compose --env-file .env down --remove-orphans -v +rm -f secrets/*.txt +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP-клиент → Nginx (8080) → API (Go, x2, stateless) + │ + ├──[S3 PUT]→ staging/ (S3, приватный bucket) + ├──[INSERT]→ Postgres: attachments (queued) + attachment_jobs (outbox) + └──[202 Accepted] + +Outbox (x1) ⇄ Postgres (attachment_jobs.published_at IS NULL) + │ + └──[XADD]→ Redis Streams (attachment.jobs) + +Worker (x2) ⇄ Redis Streams (consumer group) + ├──[SELECT ... FOR UPDATE SKIP LOCKED]→ Postgres (lease 60s) + ├──[GET]→ staging/ → resize (thumb 256, crop 128, opt) →[PUT]→ final/ + ├──[INSERT variants + UPDATE ready]→ Postgres (транзакция) + └──[XACK]→ Redis +``` + +- **Асинхронный приём (уровень 3):** `POST /api/v1/chats/{id}/attachments` возвращает **202 Accepted** сразу после того, как файл сохранён в staging, а в БД одной транзакцией записаны метаданные `attachments` (status=`queued`) и `attachment_jobs` (outbox). Это гарантирует, что после ответа 202 работа не потеряется при рестарте API. +- **Outbox (уровень 4):** отдельный процесс `cmd/outbox` читает `attachment_jobs WHERE published_at IS NULL` и публикует в брокер. `published_at` ставится только после `XADD`. Падение между COMMIT и publish приводит к повторной публикации при следующем цикле; worker идемпотентен. +- **Брокер:** Redis Streams (тот же Redis, AOF `--appendonly yes`). Consumer Group `workers`, `XACK` после durable-эффекта. В сообщении только `job_id`, `attachment_id`, `pipeline_version`, `attempt` (никаких URL или filename). +- **Worker (уровень 3-4):** отдельный процесс, реплики 2. Забирает job через `ClaimJobByID` (lease 60s). Пишет `original` (server-side copy staging→final), `thumbnail` (256×256, fit), `crop` (128×128, center-cover) и `optimized` (JPEG q=70). Всё в одной транзакции с `attachment_variants` и `status='ready'`. `XACK` только после COMMIT. +- **Идемпотентность:** ключ эффекта — `(attachment_id, pipeline_version, variant)`. Повторная доставка того же job_id: `MarkReady` — no-op (если `status='done'`), `attachment_variants` — `ON CONFLICT DO NOTHING`. Также `VariantsExist` перед Claim. +- **Retry:** `RetryJob` различает permanent (`invalid_image` → сразу failed) и transient (`storage_error` → backoff `base * 2^(n-1) + jitter`, max 5 попыток). Финальный `failed` с безопасным `error_code`. +- **Зависшие job:** `RequeueStuck` в outbox и worker, каждые 30с: `status='processing' AND lease_expires_at < now()` → `queued`, `published_at=NULL`. +- **Replay:** `POST /api/v1/attachments/{id}/replay` сбрасывает `attempt=1`, `status='queued'`, `published_at=NULL` для failed-задач. +- **Очистка:** `MoveToFinal` удаляет staging после CopyObject. `ReconcileOrphans` (в API) чистит `attachment_gc` и «зависшие» staging-объекты без job. + +### Что перенесено из lab4 + +- PostgreSQL, Argon2id, sessions (с кэшем в Redis), горизонтальное масштабирование, Nginx. +- Механизм идемпотентности `idempotency_keys` для сообщений. +- S3-хранилище и приватный bucket, `S3Store` с таймаутами и ретраями. + +### Что изменено + +- Миграция `0002_attachments.sql`: добавлены `attachment_variants`, `attachment_jobs` (outbox+lease), поля `staging_key`, `pipeline_version`, `ready_at`, `failed_at`. Таблица `attachments` теперь содержит `status IN ('queued','processing','ready','failed')`. +- `UploadAttachment` возвращает 202, статус `queued`; `DownloadAttachment` → 409 для не-ready. +- Добавлены `cmd/outbox`, `cmd/worker` (обработка изображений, `process.go`). +- `src/domain/broker.go` — обёртка Redis Streams. +- `compose.yaml`: сервисы `outbox` (1 реплика) и `worker` (2 реплики), миграции вынесены в отдельный сервис `migrate`. +- В `attachments.go`: `EnqueueAttachment`, `ClaimJob`, `ClaimJobByID`, `MarkReady`, `RetryJob`, `RequeueStuck`, `ReplayFailed`, `MarkFailedPermanent`, `GetVariantKey`, `SelectUnpublishedJobs`, `MarkPublished`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------ | +| Offline-структура и локальные refs | `make check` | PASS | [png](evidence/2_Tests_Passing.png) | +| Полная OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | [png](evidence/2_Tests_Passing.png) | +| Публичный smoke (lab5) | `make test` | PASS: 202, ready, thumbnail | [png](evidence/2_Tests_Passing.png) | +| Persistence: перезапуск API+БД+Redis+S3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | [png](evidence/2_Tests_Passing.png) | +| Failover: остановка одной реплики API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | `observed_instances_before: 2`, `successful_writes: 75`, `first_recovery: 5.371s` | [png](evidence/2_Tests_Passing.png) | +| **Ур. 3:** Upload → 202 → queued (worker не нужен) | `student_tests/states.py` | queued → ready, 3 варианта + original через `?variant=original` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Broker и worker — отдельные процессы | `docker compose ps` | `outbox`, `worker-1`, `worker-2` | [png](evidence/1_Healthcheck.png) | +| **Ур. 3:** Статусы в API, лимиты и типы | `student_tests/limits.py` | 413, 415, 400 invalid_image | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** original неизменен, thumbnail по контракту | `student_tests/variants.py` | `byte-identical`, `thumbnail 256x160 (metadata matches)`, `crop 128x128` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Broker payload без filename/URL | `student_tests/broker-payload.py` | только id/version/attempt | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Очистка staging после ready | `student_tests/staging-cleanup.py` | `staging object removed after ready` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Реестр failed (DLQ) | `student_tests/dlq.py` | `invalid_image\|1` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Идемпотентность при повторной доставке | `student_tests/redelivery.py` | `still 3 non-original variants` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Outbox закрывает разрыв COMMIT→publish | `student_tests/outbox.py` | `published_at IS NULL` при паузе outbox → ready после unpause | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Permanent error без retry | `student_tests/failed.py` | `attempt=1, invalid_image — not retried` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Transient error → retry → ready | `student_tests/retry.py` | `attempt=2`, `retry log lines: ... backoff=2s jitter=451ms` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Replay failed | `student_tests/replay.py` | `replay -> queued`, повторный permanent fail | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Lease / reaper зависших | `student_tests/lease.py` | `reaper requeued the stuck job` → `ready` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Crash после side effect до ack | `student_tests/crash-before-ack.py` | `recovery produced exactly 3 variants, no duplicates` | [txt](evidence/my_tests_passing.txt) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- | +| Остановка одной реплики API | 0 потерянных подтверждённых записей, LB переключается | Failover PASS, 75 успешных сообщений | 5.371 с, потерь нет | +| Рестарт API/БД/Redis/S3 (volumes) | Вложение и сессия живы | Persistence PASS | ~30 с (запуск контейнеров), потерь нет | +| Отказ S3 | API возвращает 503, worker ретраит | `retry.py` PASS, `attempt=2`, backoff сработал | После восстановления S3 — ready | +| Пауза outbox (разрыв COMMIT→publish) | `published_at=NULL`, upload 202 | `outbox.py` PASS, job дошёл до ready после unpause | Задержка publish = период паузы, потерь нет | +| Пауза worker (зависший processing) | Lease истекает, reaper возвращает в queued | `lease.py` PASS, job вернулся в queued через ~60с + reaper | ~90 с, потерь нет | +| `SIGKILL` worker во время processing | Повторная доставка, без дублей вариантов | `crash-before-ack.py` PASS, 3 варианта, без дублей | ~100 с (lease + рестарт), потерь нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **At-least-once:** доставка Redis Streams + идемпотентный эффект ≠ exactly-once. При падении между XADD и MarkPublished возможна повторная публикация; повторный worker это корректно обрабатывает, но сообщение в Stream продублируется. +- **Replay** доступен через endpoint `POST /api/v1/attachments/{id}/replay` (расширение сверх OpenAPI). В отчёте зафиксировано, что это не меняет семантику обязательных путей. +- **Optimized** — JPEG q=70 вместо WebP (WebP-кодирование требует cgo-зависимостей, которых в alpine-сборке без CGO нет). Контракт допускает JPEG как «перекодированный» вариант; в отчёте явно указано. +- **Broker = Redis Streams** вместо RabbitMQ — обосновано тем, что Redis уже нужен для кэша сессий, AOF включён, это уменьшает количество контейнеров. Если Redis упадёт без AOF-реплики, неопубликованные задачи будут ждать в `attachment_jobs` (outbox) — они не потеряются. +- **DLQ** — пока реестр `attachment_jobs.status='failed'` + `last_error_code`, без отдельного UI. Для оператора — `SELECT` из БД и endpoint replay. +- **Не реализовано:** версионирование нескольких pipeline (трек 5), auto-scaling worker’а по queue lag (трек 1), fault-injection с версионированием (трек 2). Уровень 5 не заявлен. +- **Нагрузка** проверена только smoke + `make test`; полноценных замеров throughput/latency pipeline не делалось (не требуется для уровня 4). diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go new file mode 100644 index 00000000..0f0cdce1 --- /dev/null +++ b/cmd/messenger/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "mtusi-messenger/src/api" + "mtusi-messenger/src/config" + "mtusi-messenger/src/domain" +) + +func main() { + cfg := config.Load() + if cfg.DatabaseURL == "" { + log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)") + } + + rootCtx, cancelRoot := context.WithCancel(context.Background()) + defer cancelRoot() + + store, err := domain.NewStore(rootCtx, cfg.DatabaseURL) + if err != nil { + log.Fatalf("postgres: %v", err) + } + defer store.Close() + + var s3store *domain.S3Store + if cfg.S3Endpoint != "" { + s3store, err = domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, + cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels) + if err != nil { + log.Fatalf("s3: %v", err) + } + } else { + log.Fatal("S3_ENDPOINT is required for lab4") + } + + // Кэш сессий в Redis/Valkey. Если CACHE_URL не задан — работаем без кэша. + var cache *domain.SessionCache + if cfg.CacheURL != "" { + cache, err = domain.NewSessionCache(rootCtx, cfg.CacheURL) + if err != nil { + if cfg.RedisRequired { + log.Fatalf("redis required but unavailable: %v", err) + } + log.Printf("redis unavailable, continuing without session cache: %v", err) + cache = nil + } else { + defer cache.Close() + } + } else if cfg.RedisRequired { + log.Fatal("SESSION_STORE=redis but CACHE_URL is not configured") + } + + sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL) + handler := &api.Handler{Store: store, Sessions: sessions, S3: s3store, MaxUpload: cfg.MaxUploadBytes} + router := api.NewRouter(handler, cfg.InstanceID) + + go func() { + ticker := time.NewTicker(2 * time.Minute) + defer ticker.Stop() + for { + select { + case <-rootCtx.Done(): + return + case <-ticker.C: + if err := store.ReconcileOrphans(rootCtx, s3store, cfg.S3GCGrace); err != nil { + log.Printf("reconcile: %v", err) + } + } + } + }() + + srv := &http.Server{ + Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + Handler: router, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + } + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + go func() { + log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL) + log.Printf("instance_id=%s", cfg.InstanceID) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("server error: %v", err) + } + }() + + <-stop + log.Println("shutting down...") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _ = srv.Shutdown(ctx) + log.Println("stopped") +} diff --git a/cmd/outbox/main.go b/cmd/outbox/main.go new file mode 100644 index 00000000..ff1bf518 --- /dev/null +++ b/cmd/outbox/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "errors" + "log" + "os" + "os/signal" + "syscall" + "time" + + "mtusi-messenger/src/config" + "mtusi-messenger/src/domain" +) + +const ( + publishInterval = 1 * time.Second + reapInterval = 30 * time.Second + publishBatch = 50 +) + +func main() { + cfg := config.Load() + if cfg.DatabaseURL == "" { + log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)") + } + if cfg.CacheURL == "" { + log.Fatal("CACHE_URL is required (env CACHE_URL or CACHE_URL_FILE)") + } + + rootCtx, cancelRoot := context.WithCancel(context.Background()) + defer cancelRoot() + + store, err := domain.NewStore(rootCtx, cfg.DatabaseURL) + if err != nil { + log.Fatalf("postgres: %v", err) + } + defer store.Close() + + broker, err := domain.NewBroker(rootCtx, cfg.CacheURL) + if err != nil { + log.Fatalf("broker: %v", err) + } + + s3, err := domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, + cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels) + if err != nil { + log.Fatalf("s3: %v", err) + } + + go gcLoop(rootCtx, store, s3, cfg.S3GCGrace) + + // Reaper зависших processing крутится здесь, потому что outbox уже + // держит соединение с БД и Redis и запускается в единственном экземпляре. + go reaperLoop(rootCtx, store) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + go func() { + <-stop + log.Println("outbox: shutting down") + cancelRoot() + }() + + ticker := time.NewTicker(publishInterval) + defer ticker.Stop() + + log.Printf("outbox: running (interval=%s batch=%d)", publishInterval, publishBatch) + for { + select { + case <-rootCtx.Done(): + log.Println("outbox: stopped") + return + case <-ticker.C: + if err := publishBatchOnce(rootCtx, store, broker); err != nil { + log.Printf("outbox: publish: %v", err) + } + } + } +} + +// publishBatchOnce: выбираем партию неопубликованных job-ов, публикуем +// по одному, после XADD — MarkPublished. Падение между XADD и MarkPublished +// приведёт к повторной публикации при следующем цикле; worker идемпотентен +// (ключ эффекта — (attachment_id, pipeline_version, variant)), поэтому +// дубль в Streams — норма, а не баг. +func publishBatchOnce(ctx context.Context, store *domain.Store, broker *domain.Broker) error { + jobs, err := store.SelectUnpublishedJobs(ctx, publishBatch) + if err != nil { + return err + } + if len(jobs) == 0 { + return nil + } + + for _, j := range jobs { + if ctx.Err() != nil { + return ctx.Err() + } + msg := domain.JobMessage{ + SchemaVersion: 1, + JobID: j.JobID, + Type: "attachment.process.v1", + AttachmentID: j.AttachmentID, + PipelineVersion: j.PipelineVersion, + Attempt: j.Attempt, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + if _, err := broker.Publish(ctx, msg); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + // Не фатально: попробуем в следующем цикле, зафиксировав причину. + _ = store.BumpPublishAttempt(ctx, j.JobID, "publish_failed") + log.Printf("outbox: publish job=%s attachment=%s: %v", + j.JobID, j.AttachmentID, err) + continue + } + if err := store.MarkPublished(ctx, j.JobID); err != nil { + log.Printf("outbox: mark published job=%s: %v", j.JobID, err) + } + } + return nil +} + +func reaperLoop(ctx context.Context, store *domain.Store) { + t := time.NewTicker(reapInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + n, err := store.RequeueStuck(ctx) + if err != nil { + log.Printf("outbox: reaper: %v", err) + continue + } + if n > 0 { + log.Printf("outbox: reaper requeued %d stuck job(s)", n) + } + } + } +} + +func gcLoop(ctx context.Context, store *domain.Store, s3 *domain.S3Store, grace time.Duration) { + t := time.NewTicker(2 * time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := store.ReconcileOrphans(ctx, s3, grace); err != nil { + log.Printf("outbox: gc: %v", err) + } + } + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..f16389b3 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "os" + "os/signal" + "syscall" + "time" + + "mtusi-messenger/src/config" + "mtusi-messenger/src/domain" + + "github.com/redis/go-redis/v9" +) + +const ( + lease = 60 * time.Second + maxAttempts = 5 + baseBackoff = 2 * time.Second +) + +func main() { + cfg := config.Load() + if cfg.CacheURL == "" { + log.Fatal("CACHE_URL (redis) is required for broker") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + store, err := domain.NewStore(ctx, cfg.DatabaseURL) + if err != nil { + log.Fatalf("db: %v", err) + } + defer store.Close() + + s3, err := domain.NewS3Store(ctx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, + cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels) + if err != nil { + log.Fatalf("s3: %v", err) + } + + broker, err := domain.NewBroker(ctx, cfg.CacheURL) + if err != nil { + log.Fatalf("broker: %v", err) + } + + consumer := cfg.InstanceID + "-worker" + + go reaperLoop(ctx, store) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-stop + log.Println("worker shutting down") + cancel() + }() + + for ctx.Err() == nil { + msgs, err := broker.Consume(ctx, consumer) + if err != nil { + if ctx.Err() != nil { + return + } + log.Printf("consume: %v", err) + time.Sleep(time.Second) + continue + } + for _, m := range msgs { + handleOne(ctx, store, s3, broker, m) + } + } +} + +func reaperLoop(ctx context.Context, store *domain.Store) { + t := time.NewTicker(30 * time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if n, err := store.RequeueStuck(ctx); err != nil { + log.Printf("reaper: %v", err) + } else if n > 0 { + log.Printf("reaper: requeued %d stuck job(s)", n) + } + } + } +} + +func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store, + broker *domain.Broker, m redis.XMessage) { + + var job domain.JobMessage + if raw, ok := m.Values["payload"].(string); ok { + _ = json.Unmarshal([]byte(raw), &job) + } else if raw, ok := m.Values["payload"].([]byte); ok { + _ = json.Unmarshal(raw, &job) + } else { + _ = broker.Ack(ctx, m.ID) + return + } + + // Валидация схемы (обязательные поля). + if job.SchemaVersion != 1 || job.Type != "attachment.process.v1" || + job.JobID == "" || job.AttachmentID == "" { + log.Printf("poison message %s, acking", m.ID) + _ = broker.Ack(ctx, m.ID) + return + } + + // No-op для уже готового результата (redelivery после готовности). + if done, _ := store.VariantsExist(ctx, job.AttachmentID, job.PipelineVersion); done { + _ = broker.Ack(ctx, m.ID) + return + } + + j, err := store.ClaimJobByID(ctx, job.JobID, workerID(), lease) + if errors.Is(err, domain.ErrNotFound) { + // Уже done/failed или lease держит кто-то другой — ack, не дублируем. + _ = broker.Ack(ctx, m.ID) + return + } + if err != nil { + log.Printf("claim: %v", err) + return + } + + stagingKey, _, err := store.GetStagingKey(ctx, j.AttachmentID) + if err != nil { + log.Printf("staging key: %v", err) + return + } + + raw, ct, err := s3.FetchStagingForProcessing(ctx, stagingKey) + if err != nil { + code, permanent := classify(err) + if permanent { + // постоянная ошибка — failed без retry + _ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, code) + } else { + _, _ = store.RetryJob(ctx, j.JobID, code, maxAttempts, baseBackoff) + } + _ = broker.Ack(ctx, m.ID) + return + } + + variants, err := processImage(raw, ct) + if err != nil { + // decode error — постоянная + _ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, "invalid_image") + _ = broker.Ack(ctx, m.ID) + return + } + + finalOriginalKey := "final/" + j.AttachmentID + "/original" + var vrows []domain.Variant + for _, v := range variants { + if v.Name == "original" { + // оригинал перекладываем из staging server-side. + if err := s3.MoveToFinal(ctx, stagingKey, finalOriginalKey); err != nil { + _, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff) + _ = broker.Ack(ctx, m.ID) + return + } + } else { + key := "final/" + j.AttachmentID + "/" + v.Name + sum, err := s3.PutBytes(ctx, key, v.ContentType, v.Data) + if err != nil { + _, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff) + _ = broker.Ack(ctx, m.ID) + return + } + v.SHA256 = sum + } + vrows = append(vrows, domain.Variant{ + Name: v.Name, ContentType: v.ContentType, SizeBytes: int64(len(v.Data)), + Width: v.Width, Height: v.Height, SHA256: v.SHA256, + }) + } + + if err := store.MarkReady(ctx, j.JobID, j.AttachmentID, j.PipelineVersion, finalOriginalKey, vrows); err != nil { + // Уже done — не страшно; иначе retry. + if !errors.Is(err, domain.ErrConflict) { + log.Printf("mark ready: %v", err) + } + _ = broker.Ack(ctx, m.ID) + return + } + _ = broker.Ack(ctx, m.ID) +} + +func workerID() string { + h, _ := os.Hostname() + return h + "-" + time.Now().Format("150405") +} + +func classify(err error) (string, bool) { + switch { + case errors.Is(err, domain.ErrTooLarge): + return "payload_too_large", true + case errors.Is(err, domain.ErrUnsupported): + return "unsupported_media_type", true + case errors.Is(err, domain.ErrNotImage): + return "invalid_image", true + default: + return "storage_error", false + } +} diff --git a/cmd/worker/process.go b/cmd/worker/process.go new file mode 100644 index 00000000..eb7500ea --- /dev/null +++ b/cmd/worker/process.go @@ -0,0 +1,137 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "image" + "image/jpeg" + "math" + + "golang.org/x/image/draw" +) + +type VariantSpec struct { + Name string + ContentType string + Width int + Height int + Data []byte + SHA256 string +} + +func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) { + var buf bytes.Buffer + if err := jpeg.Encode(&buf, src, &jpeg.Options{Quality: quality}); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// processImage возвращает 4 варианта. Оригинал — byte-identical с входом. +// thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop), +// optimized — JPEG q=70 без метаданных (WebP оставлен на усмотрение; см. отчёт). +func processImage(raw []byte, contentType string) ([]VariantSpec, error) { + src, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return nil, err + } + + out := make([]VariantSpec, 0, 4) + + // original — ровно те байты, что пришли в staging. + sum := sha256.Sum256(raw) + out = append(out, VariantSpec{ + Name: "original", ContentType: contentType, + Width: src.Bounds().Dx(), Height: src.Bounds().Dy(), + Data: raw, SHA256: hex.EncodeToString(sum[:]), + }) + + // thumbnail: fit в 256x160 с сохранением пропорций (для 16:10 — 256x160). + th := resizeFit(src, 256, 256) + if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil { + return nil, err + } + + // crop 128x128: центр-cover до меньшего квадрата. + cr := squareCover(src, 128) + if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil { + return nil, err + } + + // optimized + optBuf, err := encodeOptimizedJPEG(src, 70) + if err != nil { + return nil, err + } + optSum := sha256.Sum256(optBuf) + out = append(out, VariantSpec{ + Name: "optimized", ContentType: "image/jpeg", + Width: src.Bounds().Dx(), Height: src.Bounds().Dy(), + Data: optBuf, SHA256: hex.EncodeToString(optSum[:]), + }) + return out, nil +} + +func appendEncoded(out *[]VariantSpec, name, ct string, img image.Image) error { + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil { + return err + } + sum := sha256.Sum256(buf.Bytes()) + *out = append(*out, VariantSpec{ + Name: name, ContentType: ct, + Width: img.Bounds().Dx(), Height: img.Bounds().Dy(), + Data: buf.Bytes(), SHA256: hex.EncodeToString(sum[:]), + }) + return nil +} + +func resizeFit(src image.Image, maxW, maxH int) image.Image { + b := src.Bounds() + w, h := b.Dx(), b.Dy() + scale := float64(maxW) / float64(w) + if float64(maxH)/float64(h) < scale { + scale = float64(maxH) / float64(h) + } + if scale > 1 { + scale = 1 + } + nw := int(math.Round(float64(w) * scale)) + nh := int(math.Round(float64(h) * scale)) + dst := image.NewRGBA(image.Rect(0, 0, nw, nh)) + draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil) + return dst +} + +// squareCover масштабирует src так, чтобы покрыть квадрат size×size +// (min-масштаб по двум осям), затем центр-кроп до size×size. +// Результат — всегда ровно size×size. +func squareCover(src image.Image, size int) image.Image { + b := src.Bounds() + w, h := b.Dx(), b.Dy() + // scale — больший из двух, чтобы покрыть квадрат + scale := float64(size) / float64(w) + if float64(size)/float64(h) > scale { + scale = float64(size) / float64(h) + } + nw := int(math.Round(float64(w) * scale)) + nh := int(math.Round(float64(h) * scale)) + if nw < size { + nw = size + } + if nh < size { + nh = size + } + + scaled := image.NewRGBA(image.Rect(0, 0, nw, nh)) + draw.CatmullRom.Scale(scaled, scaled.Bounds(), src, b, draw.Over, nil) + + // center-crop до size×size + x0 := (nw - size) / 2 + y0 := (nh - size) / 2 + dst := image.NewRGBA(image.Rect(0, 0, size, size)) + draw.Draw(dst, dst.Bounds(), scaled, + image.Pt(x0, y0), draw.Src) + return dst +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..3614cc85 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,162 @@ +x-api: &api + build: + context: . + dockerfile: Dockerfile + env_file: + - .env + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + s3: { condition: service_healthy } + s3-init: { condition: service_completed_successfully } + migrate: { condition: service_completed_successfully } + networks: [backend] + +services: + postgres: + image: postgres:16.3-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: [postgres_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: { cpus: "${PG_CPUS:-1.0}", memory: "${PG_MEM:-512M}" } + + redis: + image: redis:7.2.5-alpine + command: ["redis-server", "--appendonly", "yes", "--save"] + volumes: [redisdata:/data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + networks: [backend] + + s3: + image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: [s3data:/data] + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: { cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEM:-512M}" } + + s3-init: + image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z + depends_on: + s3: { condition: service_healthy } + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + entrypoint: ["/bin/sh", "/bootstrap/bootstrap.sh"] + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro + - /tmp/silo-init-data:/data + networks: [backend] + deploy: + replicas: 1 + restart_policy: { condition: "no" } + + api: + <<: *api + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_PER_REPLICA:-0.5}" + memory: "${API_MEM_PER_REPLICA:-128M}" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 10s + timeout: 3s + start_period: 20s + retries: 5 + + outbox: + <<: *api + entrypoint: ["/app/outbox"] + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + migrate: { condition: service_completed_successfully } + deploy: + replicas: 1 + resources: + limits: { cpus: "0.2", memory: "64M" } + + worker: + <<: *api + entrypoint: ["/app/worker"] + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + s3: { condition: service_healthy } + migrate: { condition: service_completed_successfully } + deploy: + replicas: 2 + resources: + limits: { cpus: "0.5", memory: "256M" } + + migrate: + <<: *api + entrypoint: ["/app/migrate.sh"] + depends_on: + postgres: { condition: service_healthy } + deploy: + replicas: 1 + restart_policy: + condition: "no" + + lb: + image: nginx:1.27.0-alpine + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "8080:8080" + depends_on: [api] + networks: [backend] + +volumes: + pgdata: + redisdata: + s3data: + +networks: + backend: + driver: bridge + +secrets: + postgres_password: { file: ./secrets/postgres_password.txt } + database_url: { file: ./secrets/database_url.txt } + cache_url: { file: ./secrets/cache_url.txt } + s3_access_key_id: { file: ./secrets/s3_access_key_id.txt } + s3_secret_access_key: { file: ./secrets/s3_secret_access_key.txt } + minio_root_user: { file: ./secrets/minio_root_user.txt } + minio_root_password: { file: ./secrets/minio_root_password.txt } \ No newline at end of file diff --git a/evidence/1_Healthcheck.png b/evidence/1_Healthcheck.png new file mode 100644 index 00000000..951815e3 Binary files /dev/null and b/evidence/1_Healthcheck.png differ diff --git a/evidence/2_Tests_Passing.png b/evidence/2_Tests_Passing.png new file mode 100644 index 00000000..d1e3ffb2 Binary files /dev/null and b/evidence/2_Tests_Passing.png differ diff --git a/evidence/my_tests_passing.txt b/evidence/my_tests_passing.txt new file mode 100644 index 00000000..1159c9d8 --- /dev/null +++ b/evidence/my_tests_passing.txt @@ -0,0 +1,56 @@ +(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ make my_tests +python3 student_tests/states.py +upload -> queued +status stays queued while worker paused +ready with thumbnail/crop/optimized (original available via ?variant=original) +original served via ?variant=original (111817 bytes) +python3 student_tests/limits.py +413 payload_too_large +415 unsupported_media_type +415 mismatched content (client MIME not trusted) +400 invalid image (signature ok, header broken) +400 too many pixels +python3 student_tests/variants.py +original byte-identical +thumbnail 256x160 (metadata matches) +crop 128x128 JPEG +optimized present (4818 bytes) +python3 student_tests/broker-payload.py +stream sample: +1794848167433-0 +payload +{"schema_version":1,"job_id":"ee59551f-c85f-411a-b4d9-6076f31fde8f","type":"attachment.process.v1","attachment_id":"1175cf24-2984-4800-b84d-1aa702ecd7bd","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:07Z"} +1794848176445-0 +payload +{"schema_version":1,"job_id":"b7792daf-6123-4c7a-9566-b2fc4a53a614","type":"attachment.process.v1","attachment_id":"05ef10e7-19c4-47ef-9c68-a2e09eb8338e","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:16Z"} + +broker payload has only ids/version/attempt +python3 student_tests/staging-cleanup.py +staging object removed after ready (mc rc=1) +python3 student_tests/dlq.py +failed registry: +b1ccbeca-ae22-44f1-837c-868d2165889c|invalid_image|1 +python3 student_tests/redelivery.py +redelivery is no-op (still 3 non-original variants; original via object_key) +python3 student_tests/outbox.py +outbox paused: published_at IS NULL, upload accepted +after unpause: ready +python3 student_tests/failed.py +failed immediately (attempt=1), error_code=invalid_image — permanent, not retried +python3 student_tests/retry.py +transient error retried: attempt=2 +retry log lines: + worker-1 | 2026/11/16 16:56:48 retry job=64072f3d-6b9e-4101-b31b-7e9a2f51bac3 attempt=2 next_attempt=2026-11-16T16:56:50.964432673Z backoff=2s jitter=451.35473ms code=storage_error +recovered to ready after staging restored (attempt=2) +python3 student_tests/replay.py +failed reached +replay -> queued +replay reset attempt to 1; permanent error failed again — not retried +python3 student_tests/lease.py +worker stopped at processing (job=da0adcf7-64a7-427c-a50b-438e9d688bb7, lease_expires_at=2026-11-16 16:57:59.829609+00) +reaper requeued the stuck job +ready after unstuck +python3 student_tests/crash-before-ack.py +worker killed during processing +recovery produced exactly 3 non-original variants, no duplicates (original in object_key) +(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..51361002 --- /dev/null +++ b/go.mod @@ -0,0 +1,31 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.5.5 + github.com/minio/minio-go/v7 v7.0.72 + github.com/redis/go-redis/v9 v9.5.1 + golang.org/x/crypto v0.23.0 + golang.org/x/image v0.48.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rs/xid v1.5.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sync v0.25.0 // indirect + golang.org/x/sys v0.50.0 // indirect + golang.org/x/text v0.44.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..8c1b4c29 --- /dev/null +++ b/go.sum @@ -0,0 +1,64 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA= +github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik= +golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..45aed49e --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +MINIO_ROOT_USER_FILE="${MINIO_ROOT_USER_FILE:-/run/secrets/minio_root_user}" +MINIO_ROOT_PASSWORD_FILE="${MINIO_ROOT_PASSWORD_FILE:-/run/secrets/minio_root_password}" +S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}" +S3_BUCKET="${S3_BUCKET:-messenger-attachments}" +API_USER_FILE="${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}" +API_PASS_FILE="${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}" + +for f in "$MINIO_ROOT_USER_FILE" "$MINIO_ROOT_PASSWORD_FILE"; do + [ -f "$f" ] || { echo "ERROR: missing $f" >&2; exit 1; } +done + +ROOT_USER="$(cat "$MINIO_ROOT_USER_FILE")" +ROOT_PASS="$(cat "$MINIO_ROOT_PASSWORD_FILE")" + +mc alias set local "$S3_ENDPOINT" "$ROOT_USER" "$ROOT_PASS" >/dev/null + +# 1) bucket: приватный, создаётся один раз +mc mb --ignore-existing "local/$S3_BUCKET" >/dev/null +mc anonymous set none "local/$S3_BUCKET" >/dev/null + +# 2) ограниченный пользователь API (если креды заданы заранее) +if [ -s "$API_USER_FILE" ] && [ -s "$API_PASS_FILE" ]; then + API_USER="$(cat "$API_USER_FILE")" + API_PASS="$(cat "$API_PASS_FILE")" + if ! mc admin user info local "$API_USER" >/dev/null 2>&1; then + mc admin user add local "$API_USER" "$API_PASS" >/dev/null + fi + # Политика: только PutObject/GetObject/DeleteObject в bucket. + cat >/tmp/api-policy.json </dev/null 2>&1 || true + mc admin policy attach local api-attachments --user "$API_USER" >/dev/null 2>&1 || true +else + echo "WARN: API S3 credentials not pre-created; generate and place them in secrets/" >&2 +fi + +echo "s3 bootstrap done (bucket=$S3_BUCKET)" \ No newline at end of file diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..9d5c3ede --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,29 @@ +worker_processes auto; +events { worker_connections 4096; } + +http { + # Docker's embedded DNS. nginx будет обращаться сюда за IP реплик. + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 5s; + + server { + listen 8080; + client_max_body_size 12m; + + location / { + set $api_backend http://api:8080; + + proxy_pass $api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + proxy_connect_timeout 2s; + proxy_send_timeout 15s; + proxy_read_timeout 15s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + } + } +} \ No newline at end of file diff --git a/migrations/0001_core.sql b/migrations/0001_core.sql new file mode 100644 index 00000000..3dd32330 --- /dev/null +++ b/migrations/0001_core.sql @@ -0,0 +1,58 @@ +-- ============================================================ +-- Core: users, sessions, chats, members, messages, idempotency +-- ============================================================ + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id); +CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор. +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys (created_at); \ No newline at end of file diff --git a/migrations/0002_attachments.sql b/migrations/0002_attachments.sql new file mode 100644 index 00000000..d3861270 --- /dev/null +++ b/migrations/0002_attachments.sql @@ -0,0 +1,88 @@ +-- ============================================================ +-- Attachments: metadata, variants, jobs/outbox, GC +-- ============================================================ + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + object_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0 AND size_bytes <= 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width INTEGER NOT NULL CHECK (width >= 0), + height INTEGER NOT NULL CHECK (height >= 0), + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued','processing','ready','failed','pending')), + error_code TEXT, + -- поля из 0006 + pipeline_version INTEGER NOT NULL DEFAULT 1, + staging_key TEXT, + ready_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id); +-- Для reconciler/поиска «зависших» pending-записей: +CREATE INDEX IF NOT EXISTS idx_attachments_pending + ON attachments (created_at) WHERE status <> 'ready'; + +-- Связь сообщение ↔ вложения. FK на элементы массива Postgres не умеет, +-- целостность — на уровне приложения (или триггера, если нужно). +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}'; + +-- Журнал компенсации: объекты S3, которые могли остаться без метаданных. +CREATE TABLE IF NOT EXISTS attachment_gc ( + object_key TEXT PRIMARY KEY, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Idempotency эффекта: (attachment_id, pipeline_version, variant) +-- content_type без CHECK: варианты могут быть webp и т.п. +-- sha256 — text+CHECK, чтобы не ловить padding от char(64). +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + variant TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +-- Outbox + очередь job-ов +CREATE TABLE IF NOT EXISTS attachment_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued','processing','done','failed')), + -- outbox + published_at TIMESTAMPTZ, + publish_attempts INTEGER NOT NULL DEFAULT 0, + -- lease + lease_owner TEXT, + lease_expires_at TIMESTAMPTZ, + -- retry + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию +); +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished + ON attachment_jobs (next_attempt_at) + WHERE published_at IS NULL AND status = 'queued'; +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease + ON attachment_jobs (lease_expires_at) + WHERE status = 'processing'; \ No newline at end of file diff --git a/migrations/0003_messages_covering_optional.sql b/migrations/0003_messages_covering_optional.sql new file mode 100644 index 00000000..dda6537d --- /dev/null +++ b/migrations/0003_messages_covering_optional.sql @@ -0,0 +1,5 @@ +-- Индекс B для «стоимости записи»: +-- покрывающий индекс, включающий text — растёт быстрее, помогает только +-- покрывающим запросам. Накатывать отдельно, чтобы сравнить с индексом A. +CREATE INDEX IF NOT EXISTS idx_messages_chat_covering + ON messages (chat_id, created_at, id) INCLUDE (sender_id, text); \ No newline at end of file diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100644 index 00000000..4f3d4864 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +# Применяет миграции к БД, читая DATABASE_URL из файла секрета. +# Устойчиво к повторному запуску (CREATE ... IF NOT EXISTS). +URL_FILE="${DATABASE_URL_FILE:-/run/secrets/database_url}" +DATABASE_URL="$(cat "$URL_FILE")" +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1 -q" + +echo "waiting for postgres..." +for i in $(seq 1 60); do + if psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null 2>&1; then + echo "postgres is ready" + break + fi + echo " attempt $i: not ready yet" + sleep 1 +done + +psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null + +for f in /app/migrations/*.sql; do + echo "applying $f" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$f" +done +echo "migrations done" \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..742babeb --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +# ./scripts/restart.sh # 2 реплики (по умолчанию) +# ./scripts/restart.sh 4 # 4 реплики +REPLICAS="${1:-2}" + +# Останов с сохранением volumes. +docker compose down --remove-orphans + +# Пересоздание сети + контейнеров с нужным числом реплик. +API_REPLICAS="$REPLICAS" \ + docker compose --env-file .env -f compose.yaml up -d --build + +# Ждём готовности с хоста. +i=0 +while [ "$i" -lt 90 ]; do + if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then + echo "ready after ${i}s (replicas=$REPLICAS)" + exit 0 + fi + i=$((i + 1)) + sleep 1 +done +echo "stack did not become ready in 90s" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100644 index 00000000..266c2840 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu + +# Все контейнеры сервиса api (включая Exited). +# `docker compose ps -a -q api` возвращает их ID. +STOPPED="" +for id in $(docker compose ps -a -q api); do + state=$(docker inspect --format '{{.State.Status}}' "$id") + if [ "$state" = "exited" ] || [ "$state" = "created" ]; then + STOPPED="$STOPPED $id" + fi +done + +if [ -z "$STOPPED" ]; then + echo "all api containers already running" + exit 0 +fi + +for id in $STOPPED; do + hn=$(docker inspect --format '{{.Config.Hostname}}' "$id") + docker start "$id" >/dev/null + echo "started $id ($hn)" +done + +# Дождаться, пока LB увидит их снова (curl с хоста). +i=0 +while [ "$i" -lt 30 ]; do + if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then + echo "ready after ${i}s" + exit 0 + fi + i=$((i + 1)) + sleep 1 +done +echo "stack did not become ready in 30s" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..370c3476 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +# Находим ID контейнера, чьё имя (HOSTNAME) совпадает с TARGET_INSTANCE_ID. +CID=$(docker compose ps -q api | while read -r id; do + name=$(docker inspect --format '{{.Config.Hostname}}' "$id") + if [ "$name" = "$TARGET_INSTANCE_ID" ]; then + echo "$id" + break + fi +done) + +if [ -z "$CID" ]; then + echo "container with hostname $TARGET_INSTANCE_ID not found" >&2 + exit 2 +fi + +docker stop "$CID" +echo "stopped $CID ($TARGET_INSTANCE_ID)" \ No newline at end of file diff --git a/src/api/auth.go b/src/api/auth.go new file mode 100644 index 00000000..be30c192 --- /dev/null +++ b/src/api/auth.go @@ -0,0 +1,79 @@ +package api + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "strings" + + "mtusi-messenger/src/domain" +) + +// bearerAuth извлекает токен, проверяет сессию и кладёт пользователя в контекст. +// Basic на бизнес-эндпоинтах не принимается. X-User-Id игнорируется. +func (h *Handler) bearerAuth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + header := r.Header.Get("Authorization") + if !strings.HasPrefix(header, "Bearer ") { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token") + return + } + token := strings.TrimSpace(strings.TrimPrefix(header, "Bearer ")) + if token == "" { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token") + return + } + + u, err := h.Sessions.Lookup(r.Context(), token) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrUnauthorized): + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid session") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable") + return + } + + ctx := context.WithValue(r.Context(), userKey, *u) + ctx = context.WithValue(ctx, tokenKey, token) + next(w, r.WithContext(ctx)) + } +} + +func userFrom(r *http.Request) (domain.User, bool) { + u, ok := r.Context().Value(userKey).(domain.User) + return u, ok +} + +func tokenFrom(r *http.Request) string { + if v, ok := r.Context().Value(tokenKey).(string); ok { + return v + } + return "" +} + +// parseBasicCredentials разбирает Authorization: Basic base64(username:password). +func parseBasicCredentials(header string) (string, string, error) { + if !strings.HasPrefix(header, "Basic ") { + return "", "", errors.New("not basic") + } + raw := strings.TrimSpace(strings.TrimPrefix(header, "Basic ")) + decoded, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return "", "", errors.New("invalid base64") + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", errors.New("missing colon") + } + return parts[0], parts[1], nil +} diff --git a/src/api/errors.go b/src/api/errors.go new file mode 100644 index 00000000..7c5b5df8 --- /dev/null +++ b/src/api/errors.go @@ -0,0 +1,28 @@ +package api + +import ( + "encoding/json" + "net/http" + + "mtusi-messenger/src/domain" +) + +// writeError пишет ошибку в формате контракта. +func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(domain.ErrorBody{ + Error: domain.ErrorDetail{ + Code: code, + Message: message, + RequestID: requestIDFrom(r), + }, + }) +} + +// writeJSON пишет успешный JSON-ответ. +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/src/api/handlers.go b/src/api/handlers.go new file mode 100644 index 00000000..4b63ec97 --- /dev/null +++ b/src/api/handlers.go @@ -0,0 +1,603 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + + "mtusi-messenger/src/domain" + + "github.com/google/uuid" +) + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type Handler struct { + Store *domain.Store + Sessions *domain.SessionStore + S3 *domain.S3Store + MaxUpload int64 +} + +// --- Health --- + +func (h *Handler) Live(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +// Ready проверяет БД (обязательная зависимость). Redis — кэш, его недоступность +// не должна ронять readiness: API может работать, обходя кэш. +func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) { + if err := h.Store.Ping(r.Context()); err != nil { + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +// --- Users --- + +type createUserReq struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) { + var req createUserReq + if err := decodeSingleJSON(r.Body, &req); err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") + return + } + if !usernameRe.MatchString(req.Username) { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid username") + return + } + if n := len([]rune(req.DisplayName)); n < 1 || n > 100 || strings.TrimSpace(req.DisplayName) == "" { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid display_name") + return + } + // Пароль: 12–128 code points, не состоит только из пробелов. + if n := len([]rune(req.Password)); n < 12 || n > 128 || strings.TrimSpace(req.Password) == "" { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid password") + return + } + u, err := h.Store.CreateUser(r.Context(), req.Username, req.DisplayName, req.Password) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrConflict): + writeError(w, r, http.StatusConflict, "conflict", "username taken") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusCreated, u) +} + +func (h *Handler) CurrentUser(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + u, _ := userFrom(r) + writeJSON(w, http.StatusOK, u) + })(w, r) +} + +// --- Auth --- + +func (h *Handler) CreateSession(w http.ResponseWriter, r *http.Request) { + username, password, err := parseBasicCredentials(r.Header.Get("Authorization")) + if err != nil { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials") + return + } + u, err := h.Store.VerifyPassword(r.Context(), username, password) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrUnauthorized): + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + s, err := h.Sessions.Create(r.Context(), *u) + if err != nil { + if errors.Is(err, domain.ErrTimeout) { + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout") + return + } + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable") + return + } + writeJSON(w, http.StatusCreated, s) +} + +func (h *Handler) RevokeSession(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + token := tokenFrom(r) + if err := h.Sessions.Revoke(r.Context(), token); err != nil { + if errors.Is(err, domain.ErrTimeout) { + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout") + return + } + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable") + return + } + w.WriteHeader(http.StatusNoContent) + })(w, r) +} + +// --- Chats --- + +type createChatReq struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +func (h *Handler) CreateChat(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + var req createChatReq + if err := decodeSingleJSON(r.Body, &req); err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") + return + } + if strings.TrimSpace(req.Title) == "" || len([]rune(req.Title)) > 100 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid title") + return + } + if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid member_ids") + return + } + c, err := h.Store.CreateChat(r.Context(), req.Title, req.MemberIDs, me.ID) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid members") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusCreated, c) + })(w, r) +} + +func (h *Handler) ListChats(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + limit := 50 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit") + return + } + limit = n + } + cursor := r.URL.Query().Get("cursor") + chats, next, err := h.Store.ListChats(r.Context(), me.ID, limit, cursor) + switch { + case err == nil: + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusOK, domain.ChatPage{Items: chats, NextCursor: next}) + })(w, r) +} + +func (h *Handler) GetChat(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + chatID := r.PathValue("chat_id") + c, err := h.Store.GetChat(r.Context(), chatID, me.ID) + switch { + case err == nil: + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id") + return + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "chat not found") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusOK, c) + })(w, r) +} + +// --- Messages --- + +type createMessageReq struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + chatID := r.PathValue("chat_id") + + // Idempotency-Key обязателен. + key := r.Header.Get("Idempotency-Key") + if key == "" { + writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required") + return + } + if !idempotencyKeyRe.MatchString(key) { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key") + return + } + + if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil { + switch { + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id") + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "chat not found") + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } + return + } + + // Читаем тело один раз и хешируем нормализованный JSON. + rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024)) + if err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body") + return + } + var req createMessageReq + if err := decodeSingleJSON(bytes.NewReader(rawBody), &req); err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") + return + } + if len([]rune(req.Text)) > 4000 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "message too long") + return + } + if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message") + return + } + bodyHash, err := domain.HashBody(rawBody) + if err != nil { + writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON") + return + } + if len(req.Text) > 0 && len([]rune(req.Text)) == 0 { + // всё пробелы, но вложения есть — допустимо + } + + attachmentIDs := req.AttachmentIDs + if attachmentIDs == nil { + attachmentIDs = []string{} + } + msg, rec, replay, err := h.Store.AddMessageIdempotent( + r.Context(), chatID, me.ID, req.Text, attachmentIDs, key, bodyHash) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "chat not found") + return + case errors.Is(err, domain.ErrConflict): + writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + + if replay { + // Возвращаем сохранённый 201 и то же Message. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(rec.StatusCode) + _, _ = w.Write(rec.Response) + return + } + writeJSON(w, http.StatusCreated, msg) + })(w, r) +} + +func (h *Handler) ListMessages(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + chatID := r.PathValue("chat_id") + if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil { + switch { + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id") + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "chat not found") + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } + return + } + limit := 50 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit") + return + } + limit = n + } + cursor := r.URL.Query().Get("cursor") + items, next, err := h.Store.ListMessages(r.Context(), chatID, limit, cursor) + switch { + case err == nil: + // ok + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor") + return + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "query timeout") + return + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusOK, domain.MessagePage{Items: items, NextCursor: next}) + })(w, r) +} + +func decodeSingleJSON(r io.Reader, v any) error { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if dec.More() { + return errors.New("trailing data after JSON value") + } + // Пробуем прочитать ещё один токен — если это не EOF, значит есть мусор. + var extra any + if err := dec.Decode(&extra); err != io.EOF { + return errors.New("trailing data after JSON value") + } + return nil +} + +// --- Attachments --- + +// safeFilename отбрасывает пути и опасные имена; используется только для отображения. +func safeFilename(name string) (string, bool) { + name = strings.TrimSpace(name) + if name == "" || len(name) > 255 { + return "", false + } + if strings.ContainsAny(name, "/\\") || strings.Contains(name, "..") { + return "", false + } + for _, r := range name { + if r < 0x20 || r == 0x7f { + return "", false + } + } + return name, true +} + +func errorCodeFor(err error) string { + switch { + case errors.Is(err, domain.ErrTooLarge): + return "payload_too_large" + case errors.Is(err, domain.ErrUnsupported): + return "unsupported_media_type" + case errors.Is(err, domain.ErrNotImage): + return "invalid_image" + default: + return "storage_error" + } +} + +func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + chatID := r.PathValue("chat_id") + + if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil { + switch { + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id") + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "chat not found") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } + return + } + + filename, ok := safeFilename(r.URL.Query().Get("filename")) + if !ok { + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid filename") + return + } + + stagingKey := domain.StagingPrefix + chatID + "/" + uuid.NewString() + pendingID, err := h.Store.CreateAttachmentPending( + r.Context(), chatID, me.ID, stagingKey, filename) + if err != nil { + switch { + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid attachment metadata") + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } + return + } + + res, err := h.S3.PutValidatedImage(r.Context(), stagingKey, r.Body) + if err != nil { + _ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, errorCodeFor(err)) + _ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "upload_failed") + switch { + case errors.Is(err, domain.ErrTooLarge): + writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large") + case errors.Is(err, domain.ErrUnsupported): + writeError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "only PNG/JPEG") + case errors.Is(err, domain.ErrNotImage): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid image") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable") + } + return + } + + // Атомарно: attachment queued + outbox-строка. 202 только после COMMIT. + att, err := h.Store.EnqueueAttachment( + r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256, + res.Width, res.Height, domain.PipelineVersion) + if err != nil { + _ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "enqueue_failed") + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + writeJSON(w, http.StatusAccepted, att) + })(w, r) +} + +func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + id := r.PathValue("attachment_id") + att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID) + switch { + case err == nil: + writeJSON(w, http.StatusOK, att) + case errors.Is(err, domain.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not_found", "attachment not found") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } + })(w, r) +} + +func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + me, _ := userFrom(r) + id := r.PathValue("attachment_id") + att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID) + if err != nil { + if errors.Is(err, domain.ErrNotFound) { + writeError(w, r, http.StatusNotFound, "not_found", "attachment not found") + return + } + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + switch att.Status { + case domain.AttachmentReady: + // ok + case domain.AttachmentFailed: + writeError(w, r, http.StatusConflict, "conflict", "attachment failed") + return + default: + writeError(w, r, http.StatusConflict, "conflict", "attachment not ready") + return + } + + variant := r.URL.Query().Get("variant") + if variant == "" { + variant = "original" + } + var key string + if variant == "original" { + key, err = h.Store.GetAttachmentObjectKey(r.Context(), att.ID) + } else { + key, _, err = h.Store.GetVariantKey(r.Context(), att.ID, att.PipelineVersion, variant) + } + if err != nil { + writeError(w, r, http.StatusNotFound, "not_found", "attachment not found") + return + } + + rc, contentType, size, err := h.S3.GetObject(r.Context(), key) + if err != nil { + if errors.Is(err, domain.ErrNotFound) { + writeError(w, r, http.StatusNotFound, "not_found", "file missing in storage") + return + } + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable") + return + } + defer rc.Close() + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Disposition", `inline; filename="`+strings.ReplaceAll(att.Filename, `"`, "")+`"`) + w.WriteHeader(http.StatusOK) + + written, err := io.Copy(w, rc) + if err != nil || written != size { + // Соединение уже частично отправлено, но клиент получит обрезанный файл. + // Логируем это для отладки. + log.Printf("download truncated: written=%d expected=%d err=%v", written, size, err) + } + })(w, r) +} + +func (h *Handler) ReplayAttachment(w http.ResponseWriter, r *http.Request) { + h.bearerAuth(func(w http.ResponseWriter, r *http.Request) { + // Только owner или участник чата; упрощённо — участник чата. + me, _ := userFrom(r) + id := r.PathValue("attachment_id") + if _, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID); err != nil { + writeError(w, r, http.StatusNotFound, "not_found", "attachment not found") + return + } + err := h.Store.ReplayFailed(r.Context(), id) + if errors.Is(err, domain.ErrConflict) { + writeError(w, r, http.StatusConflict, "conflict", "not in failed state") + return + } + if err != nil { + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + return + } + w.WriteHeader(http.StatusAccepted) + })(w, r) +} diff --git a/src/api/middleware.go b/src/api/middleware.go new file mode 100644 index 00000000..7cce92e1 --- /dev/null +++ b/src/api/middleware.go @@ -0,0 +1,46 @@ +package api + +import ( + "context" + "net/http" + + "github.com/google/uuid" +) + +type ctxKey string + +const ( + requestIDKey ctxKey = "request_id" + userKey ctxKey = "user" + tokenKey ctxKey = "token" +) + +func requestIDFrom(r *http.Request) string { + if v, ok := r.Context().Value(requestIDKey).(string); ok { + return v + } + return "" +} + +func withRequestID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-Request-Id") + if id == "" { + id = uuid.NewString() + } + w.Header().Set("X-Request-Id", id) + ctx := context.WithValue(r.Context(), requestIDKey, id) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health). +// ID генерируется один раз при старте процесса — стабилен в пределах жизни. +func withInstanceID(instanceID string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", instanceID) + next.ServeHTTP(w, r) + }) + } +} diff --git a/src/api/router.go b/src/api/router.go new file mode 100644 index 00000000..f82af911 --- /dev/null +++ b/src/api/router.go @@ -0,0 +1,25 @@ +package api + +import "net/http" + +func NewRouter(h *Handler, instanceID string) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", h.Live) + mux.HandleFunc("GET /health/ready", h.Ready) + mux.HandleFunc("POST /api/v1/users", h.CreateUser) + mux.HandleFunc("GET /api/v1/users/me", h.CurrentUser) + mux.HandleFunc("POST /api/v1/auth/sessions", h.CreateSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", h.RevokeSession) + mux.HandleFunc("POST /api/v1/chats", h.CreateChat) + mux.HandleFunc("GET /api/v1/chats", h.ListChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment) + mux.HandleFunc("POST /api/v1/attachments/{attachment_id}/replay", h.ReplayAttachment) + + // Порядок: instanceID снаружи, requestID внутри. + return withInstanceID(instanceID)(withRequestID(mux)) +} diff --git a/src/config/config.go b/src/config/config.go new file mode 100644 index 00000000..d9fa3e8c --- /dev/null +++ b/src/config/config.go @@ -0,0 +1,136 @@ +package config + +import ( + "crypto/rand" + "encoding/hex" + "os" + "strconv" + "time" +) + +type Config struct { + Host string + Port int + Env string + DatabaseURL string + CacheURL string + SessionTTL time.Duration + SessionStore string + RedisRequired bool + InstanceID string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKeyID string + S3SecretKey string + MaxUploadBytes int64 + MaxImagePixels int64 + S3GCGrace time.Duration +} + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func readSecretFromFile(path string) string { + if path == "" { + return "" + } + b, err := os.ReadFile(path) + if err != nil { + return "" + } + return string(b) +} + +func Load() Config { + port := 8080 + if v := os.Getenv("APP_PORT"); v != "" { + if p, err := strconv.Atoi(v); err == nil { + port = p + } + } + ttl := 3600 + if v := os.Getenv("SESSION_TTL_SECONDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + ttl = n + } + } + + // DATABASE_URL и CACHE_URL могут прийти либо напрямую, либо через *_FILE. + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + dbURL = readSecretFromFile(os.Getenv("DATABASE_URL_FILE")) + } + cacheURL := os.Getenv("CACHE_URL") + if cacheURL == "" { + cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE")) + } + + instanceID := os.Getenv("INSTANCE_ID") + if instanceID == "" { + instanceID = os.Getenv("HOSTNAME") + } + if instanceID == "" { + instanceID = "api-" + randomShortID() + } + + store := getenv("SESSION_STORE", "postgres") + + s3Access := os.Getenv("S3_ACCESS_KEY_ID") + if s3Access == "" { + s3Access = readSecretFromFile(os.Getenv("S3_ACCESS_KEY_ID_FILE")) + } + s3Secret := os.Getenv("S3_SECRET_ACCESS_KEY") + if s3Secret == "" { + s3Secret = readSecretFromFile(os.Getenv("S3_SECRET_ACCESS_KEY_FILE")) + } + maxUpload := int64(10485760) + if v := os.Getenv("MAX_UPLOAD_BYTES"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + maxUpload = n + } + } + maxPixels := int64(40000000) + if v := os.Getenv("MAX_IMAGE_PIXELS"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + maxPixels = n + } + } + gcGrace := 600 * time.Second + if v := os.Getenv("S3_GC_GRACE_SECONDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + gcGrace = time.Duration(n) * time.Second + } + } + return Config{ + Host: getenv("APP_HOST", "0.0.0.0"), + Port: port, + Env: getenv("APP_ENV", "lab"), + DatabaseURL: dbURL, + CacheURL: cacheURL, + SessionTTL: time.Duration(ttl) * time.Second, + SessionStore: store, + RedisRequired: store == "redis", + InstanceID: instanceID, + S3Endpoint: getenv("S3_ENDPOINT", ""), + S3Region: getenv("S3_REGION", "us-east-1"), + S3Bucket: getenv("S3_BUCKET", "messenger-attachments"), + S3AccessKeyID: s3Access, + S3SecretKey: s3Secret, + MaxUploadBytes: maxUpload, + MaxImagePixels: maxPixels, + S3GCGrace: gcGrace, + } +} + +func randomShortID() string { + b := make([]byte, 6) + if _, err := rand.Read(b); err != nil { + return "unknown" + } + return hex.EncodeToString(b) +} diff --git a/src/domain/attachment_states.go b/src/domain/attachment_states.go new file mode 100644 index 00000000..15e27aa8 --- /dev/null +++ b/src/domain/attachment_states.go @@ -0,0 +1,12 @@ +package domain + +const ( + AttachmentQueued = "queued" + AttachmentProcessing = "processing" + AttachmentReady = "ready" + AttachmentFailed = "failed" +) + +const PipelineVersion = 1 + +var RequiredVariants = []string{"thumbnail", "crop", "optimized"} diff --git a/src/domain/attachments.go b/src/domain/attachments.go new file mode 100644 index 00000000..dd9e8137 --- /dev/null +++ b/src/domain/attachments.go @@ -0,0 +1,613 @@ +package domain + +import ( + "context" + "errors" + "log" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// CreateAttachmentPending резервирует строку ДО записи в S3. +// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится. +func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, stagingKey, filename string) (string, error) { + id := uuid.NewString() + _, err := s.pool.Exec(ctx, + `INSERT INTO attachments (id, chat_id, owner_id, object_key, staging_key, filename, + content_type, size_bytes, sha256, width, height, + pipeline_version, status) + VALUES ($1,$2,$3,$4,$4,$5,'application/octet-stream',0,repeat('0',64),0,0,1,'queued')`, + id, chatID, ownerID, stagingKey, filename) + if err != nil { + return "", classifyDBError(err) + } + return id, nil +} + +// MarkAttachmentFailed помечает pending как failed (компенсация). +func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error { + _, err := s.pool.Exec(ctx, + `UPDATE attachments SET status='failed', error_code=$2, updated_at=now() + WHERE id=$1 AND status='queued'`, id, code) + return classifyDBError(err) +} + +// ScheduleObjectGC ставит object_key в очередь на удаление (grace period). +func (s *Store) ScheduleObjectGC(ctx context.Context, objectKey, reason string) error { + _, err := s.pool.Exec(ctx, + `INSERT INTO attachment_gc (object_key, reason) VALUES ($1,$2) + ON CONFLICT (object_key) DO NOTHING`, objectKey, reason) + return classifyDBError(err) +} + +// GetAttachmentForUser возвращает вложение, только если пользователь — участник чата. +func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*Attachment, error) { + var a Attachment + var errCode *string + err := s.pool.QueryRow(ctx, ` + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id + WHERE a.id=$1 AND m.user_id=$2`, id, userID, + ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, + &a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + a.ErrorCode = errCode + a.Variants = []Variant{} + rows, err := s.pool.Query(ctx, + `SELECT variant, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id=$1 AND pipeline_version=$2 + AND variant = ANY($3) + ORDER BY variant`, + a.ID, a.PipelineVersion, RequiredVariants) + if err != nil { + return nil, classifyDBError(err) + } + defer rows.Close() + for rows.Next() { + var v Variant + if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, + &v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil { + return nil, classifyDBError(err) + } + a.Variants = append(a.Variants, v) + } + if err := rows.Err(); err != nil { + return nil, classifyDBError(err) + } + return &a, nil +} + +// GetAttachmentObjectKey — только для внутреннего download после проверки ACL. +func (s *Store) GetAttachmentObjectKey(ctx context.Context, id string) (string, error) { + var key string + err := s.pool.QueryRow(ctx, `SELECT object_key FROM attachments WHERE id=$1`, id).Scan(&key) + if errors.Is(err, pgx.ErrNoRows) { + return "", ErrNotFound + } + if err != nil { + return "", classifyDBError(err) + } + return key, nil +} + +// ValidateAttachmentIDsForChat проверяет, что все вложения принадлежат чату, +// имеют статус ready и лимит 10. Возвращает ErrInvalid/ErrNotFound. +func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string, ids []string) error { + if len(ids) > 10 { + return ErrInvalid + } + seen := map[string]bool{} + for _, id := range ids { + if seen[id] { + return ErrInvalid + } + seen[id] = true + var status string + err := s.pool.QueryRow(ctx, + `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`, + id, chatID).Scan(&status) + if err == nil && status == AttachmentFailed { + return ErrInvalid + } + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return classifyDBError(err) + } + } + return nil +} + +// ReconcileOrphans: удаляет объекты из attachment_gc старше grace, которых нет в БД, +// и «призрачные» объекты pending старше grace. Вызывается из фонового тикера. +func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Duration) error { + // 1) GC-очередь: удалить объекты в S3, затем строку. + rows, err := s.pool.Query(ctx, + `SELECT object_key FROM attachment_gc WHERE created_at < now() - $1::interval LIMIT 100`, + grace.String()) + if err != nil { + return classifyDBError(err) + } + var keys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return classifyDBError(err) + } + keys = append(keys, k) + } + rows.Close() + for _, k := range keys { + if err := s3.DeleteObject(ctx, k); err != nil { + continue + } + _, _ = s.pool.Exec(ctx, `DELETE FROM attachment_gc WHERE object_key=$1`, k) + } + + // 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет. + prow, err := s.pool.Query(ctx, + `SELECT a.id, a.staging_key FROM attachments a + LEFT JOIN attachment_jobs j ON j.attachment_id = a.id + WHERE a.status='queued' AND j.job_id IS NULL + AND a.created_at < now() - $1::interval LIMIT 100`, + grace.String()) + if err != nil { + return classifyDBError(err) + } + type pend struct{ id, key string } + var pendings []pend + for prow.Next() { + var p pend + if err := prow.Scan(&p.id, &p.key); err != nil { + prow.Close() + return classifyDBError(err) + } + pendings = append(pendings, p) + } + prow.Close() + for _, p := range pendings { + _ = s3.DeleteObject(ctx, p.key) + _, _ = s.pool.Exec(ctx, `DELETE FROM attachments WHERE id=$1 AND status='pending'`, p.id) + } + return nil +} + +// EnqueueAttachment: атомарно фиксируем метаданные приёма и кладём outbox-строку. +// 202 отвечаем только после успешного COMMIT этой транзакции. +func (s *Store) EnqueueAttachment( + ctx context.Context, attachmentID, contentType string, + size int64, sha string, w, h int, pipelineVersion int, +) (*Attachment, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, classifyDBError(err) + } + defer tx.Rollback(ctx) + + var a Attachment + err = tx.QueryRow(ctx, ` + UPDATE attachments + SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6, + pipeline_version=$7, status='queued', updated_at=now() + WHERE id=$1 AND status='queued' + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, error_code, created_at, updated_at, pipeline_version`, + attachmentID, contentType, size, sha, w, h, pipelineVersion, + ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, + &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + + if _, err := tx.Exec(ctx, ` + INSERT INTO attachment_jobs + (job_id, attachment_id, pipeline_version, status, next_attempt_at) + VALUES ($1,$2,$3,'queued', now())`, + uuid.NewString(), attachmentID, pipelineVersion, + ); err != nil { + return nil, classifyDBError(err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, classifyDBError(err) + } + a.Variants = []Variant{} + return &a, nil +} + +// ClaimJob — берём одну доступную задачу под lease. +// Работает и для queued, и для протухшего processing (reaper тем же запросом). +func (s *Store) ClaimJob(ctx context.Context, workerID string, lease time.Duration) (*Job, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, classifyDBError(err) + } + defer tx.Rollback(ctx) + + var j Job + err = tx.QueryRow(ctx, ` + WITH picked AS ( + SELECT job_id FROM attachment_jobs + WHERE status IN ('queued','processing') + AND next_attempt_at <= now() + AND (status='queued' OR lease_expires_at < now()) + ORDER BY next_attempt_at + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + UPDATE attachment_jobs aj + SET status='processing', + lease_owner=$1, + lease_expires_at = now() + $2::interval, + updated_at = now() + FROM picked + WHERE aj.job_id = picked.job_id + RETURNING aj.job_id, aj.attachment_id, aj.pipeline_version, + aj.attempt, aj.status, aj.next_attempt_at, aj.last_error_code`, + workerID, lease.String(), + ).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.Status, + &j.NextAttemptAt, &j.LastError) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, classifyDBError(err) + } + return &j, nil +} + +// MarkReady: атомарно пишем все варианты (идемпотентно по ключу) и завершаем job. +// Конкурентная попытка не перезапишет существующий вариант — ON CONFLICT DO NOTHING. +func (s *Store) MarkReady( + ctx context.Context, jobID, attachmentID string, pipelineVersion int, + finalOriginalKey string, variants []Variant, +) error { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + return classifyDBError(err) + } + defer tx.Rollback(ctx) + + // Если job уже done — no-op (redelivery после ack). + var status string + err = tx.QueryRow(ctx, + `SELECT status FROM attachment_jobs WHERE job_id=$1 FOR UPDATE`, jobID, + ).Scan(&status) + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return classifyDBError(err) + } + if status == "done" { + return tx.Commit(ctx) + } + if status == "failed" { + return ErrConflict // нельзя откатывать failed→ready; см. replay + } + + // Записать оригинал в final prefix. + if _, err := tx.Exec(ctx, + `UPDATE attachments + SET object_key=$2, status='ready', ready_at=now(), updated_at=now() + WHERE id=$1 AND pipeline_version=$3`, + attachmentID, finalOriginalKey, pipelineVersion, + ); err != nil { + return classifyDBError(err) + } + + for _, v := range variants { + if v.Name == "original" { + continue // original живёт в attachments.object_key, не в variants + } + if _, err := tx.Exec(ctx, ` + INSERT INTO attachment_variants + (attachment_id, pipeline_version, variant, object_key, + content_type, size_bytes, width, height, sha256) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`, + attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name, + v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256, + ); err != nil { + return classifyDBError(err) + } + } + + if _, err := tx.Exec(ctx, + `UPDATE attachment_jobs + SET status='done', lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE job_id=$1`, jobID, + ); err != nil { + return classifyDBError(err) + } + return tx.Commit(ctx) +} + +// RetryJob: временная ошибка — вернуть в очередь с backoff+jitter. +// exhausted=true — если попытки кончились. +func (s *Store) RetryJob( + ctx context.Context, jobID string, errorCode string, + maxAttempts int, baseBackoff time.Duration, +) (exhausted bool, err error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return false, classifyDBError(err) + } + defer tx.Rollback(ctx) + + var attempt int + var attachmentID string + err = tx.QueryRow(ctx, + `SELECT attempt, attachment_id FROM attachment_jobs + WHERE job_id=$1 FOR UPDATE`, jobID, + ).Scan(&attempt, &attachmentID) + if err != nil { + return false, classifyDBError(err) + } + + if attempt >= maxAttempts { + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status='failed', last_error_code=$2, + lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE job_id=$1`, jobID, errorCode); err != nil { + return false, classifyDBError(err) + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status='failed', failed_at=now(), + error_code=$2, updated_at=now() + WHERE id=$1`, attachmentID, errorCode); err != nil { + return false, classifyDBError(err) + } + if err := tx.Commit(ctx); err != nil { + return false, classifyDBError(err) + } + log.Printf("retry job=%s exhausted attempts=%d code=%s", + jobID, attempt, errorCode) + return true, nil + } + + // backoff: base * 2^(attempt-1) + jitter + backoff := baseBackoff * time.Duration(1<<(attempt-1)) + jitter := time.Duration(rand.Int63n(int64(backoff / 4))) + next := time.Now().Add(backoff + jitter) + + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET attempt = attempt + 1, + status = 'queued', + last_error_code=$2, + next_attempt_at=$3, + published_at=NULL, + lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE job_id=$1`, jobID, errorCode, next); err != nil { + return false, classifyDBError(err) + } + if err := tx.Commit(ctx); err != nil { + return false, classifyDBError(err) + } + // Логируем зафиксированный retry — используется в student-tests/backoff.py + log.Printf("retry job=%s attempt=%d next_attempt=%s backoff=%s jitter=%s code=%s", + jobID, attempt+1, next.UTC().Format(time.RFC3339Nano), + backoff, jitter, errorCode) + + return false, nil +} + +// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox. +func (s *Store) RequeueStuck(ctx context.Context) (int64, error) { + tag, err := s.pool.Exec(ctx, ` + UPDATE attachment_jobs + SET status='queued', lease_owner=NULL, lease_expires_at=NULL, + next_attempt_at = now(), published_at = NULL, updated_at = now() + WHERE status='processing' AND lease_expires_at < now()`) + if err != nil { + return 0, classifyDBError(err) + } + return tag.RowsAffected(), nil +} + +// ReplayFailed — операторский replay (уровень 4, CLI/endpoint). +func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return classifyDBError(err) + } + defer tx.Rollback(ctx) + + tag, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status='queued', attempt=1, next_attempt_at=now(), + published_at=NULL, + last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE attachment_id=$1 AND status='failed'`, attachmentID) + if err != nil { + return classifyDBError(err) + } + if tag.RowsAffected() == 0 { + return ErrConflict + } + + if _, err := tx.Exec(ctx, + `UPDATE attachments SET status='queued', error_code=NULL, failed_at=NULL, + updated_at=now() + WHERE id=$1 AND status='failed'`, attachmentID); err != nil { + return classifyDBError(err) + } + return tx.Commit(ctx) +} + +// ClaimJobByID — берёт конкретную job под lease. Нужен для redelivery: +// если worker упал после publish, повторно доставленное сообщение должно +// либо взять ту же работу (lease протух), либо быть no-op (status=done). +func (s *Store) ClaimJobByID(ctx context.Context, jobID, workerID string, lease time.Duration) (*Job, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, classifyDBError(err) + } + defer tx.Rollback(ctx) + + var j Job + err = tx.QueryRow(ctx, ` + UPDATE attachment_jobs + SET status='processing', + lease_owner=$2, + lease_expires_at = now() + $3::interval, + updated_at = now() + WHERE job_id=$1 + AND status IN ('queued','processing') + AND (status='queued' OR lease_expires_at < now()) + RETURNING job_id, attachment_id, pipeline_version, attempt, + status, next_attempt_at, last_error_code`, + jobID, workerID, lease.String(), + ).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, + &j.Status, &j.NextAttemptAt, &j.LastError) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, classifyDBError(err) + } + return &j, nil +} + +// GetStagingKey — доверенные метаданные для worker’а (не из сообщения брокера). +func (s *Store) GetStagingKey(ctx context.Context, attachmentID string) (string, string, error) { + var key, ct string + err := s.pool.QueryRow(ctx, + `SELECT staging_key, content_type FROM attachments WHERE id=$1`, attachmentID, + ).Scan(&key, &ct) + if errors.Is(err, pgx.ErrNoRows) { + return "", "", ErrNotFound + } + if err != nil { + return "", "", classifyDBError(err) + } + return key, ct, nil +} + +// VariantsExist — no-op-проверка на повторную доставку уже готовой работы. +func (s *Store) VariantsExist(ctx context.Context, attachmentID string, pv int) (bool, error) { + var n int + err := s.pool.QueryRow(ctx, + `SELECT count(*) FROM attachment_variants + WHERE attachment_id=$1 AND pipeline_version=$2`, + attachmentID, pv, + ).Scan(&n) + if err != nil { + return false, classifyDBError(err) + } + return n >= len(RequiredVariants), nil +} + +// MarkFailedPermanent — терминальный failed без retry (постоянная ошибка). +func (s *Store) MarkFailedPermanent(ctx context.Context, jobID, attachmentID, code string) error { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return classifyDBError(err) + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, + `UPDATE attachment_jobs SET status='failed', last_error_code=$2, + lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE job_id=$1 AND status='processing'`, jobID, code); err != nil { + return classifyDBError(err) + } + if _, err := tx.Exec(ctx, + `UPDATE attachments SET status='failed', error_code=$2, + failed_at=now(), updated_at=now() + WHERE id=$1`, attachmentID, code); err != nil { + return classifyDBError(err) + } + return tx.Commit(ctx) +} + +type UnpublishedJob struct { + JobID string + AttachmentID string + PipelineVersion int + Attempt int +} + +// SelectUnpublishedJobs — выборка outbox’ом. Транзакция не нужна: +// published_at ставим отдельным UPDATE после XADD; повторная публикация +// допустима (worker идемпотентен). +func (s *Store) SelectUnpublishedJobs(ctx context.Context, limit int) ([]UnpublishedJob, error) { + rows, err := s.pool.Query(ctx, ` + SELECT job_id, attachment_id, pipeline_version, attempt + FROM attachment_jobs + WHERE published_at IS NULL AND status='queued' + AND next_attempt_at <= now() + ORDER BY next_attempt_at + LIMIT $1`, limit) + if err != nil { + return nil, classifyDBError(err) + } + defer rows.Close() + var out []UnpublishedJob + for rows.Next() { + var u UnpublishedJob + if err := rows.Scan(&u.JobID, &u.AttachmentID, &u.PipelineVersion, &u.Attempt); err != nil { + return nil, classifyDBError(err) + } + out = append(out, u) + } + return out, rows.Err() +} + +// MarkPublished — фиксируем факт XADD (publisher confirm). +func (s *Store) MarkPublished(ctx context.Context, jobID string) error { + _, err := s.pool.Exec(ctx, + `UPDATE attachment_jobs SET published_at=now(), + publish_attempts=publish_attempts+1, updated_at=now() + WHERE job_id=$1`, jobID) + return classifyDBError(err) +} + +// BumpPublishAttempt — XADD упал; фиксируем причину, повторим в следующем цикле. +func (s *Store) BumpPublishAttempt(ctx context.Context, jobID, code string) error { + _, err := s.pool.Exec(ctx, + `UPDATE attachment_jobs SET publish_attempts=publish_attempts+1, + last_error_code=$2, updated_at=now() + WHERE job_id=$1`, jobID, code) + return classifyDBError(err) +} + +func (s *Store) GetVariantKey(ctx context.Context, attachmentID string, pv int, variant string) (string, string, error) { + var key, ct string + err := s.pool.QueryRow(ctx, + `SELECT object_key, content_type FROM attachment_variants + WHERE attachment_id=$1 AND pipeline_version=$2 AND variant=$3`, + attachmentID, pv, variant).Scan(&key, &ct) + if errors.Is(err, pgx.ErrNoRows) { + return "", "", ErrNotFound + } + if err != nil { + return "", "", classifyDBError(err) + } + return key, ct, nil +} diff --git a/src/domain/broker.go b/src/domain/broker.go new file mode 100644 index 00000000..4e2b728d --- /dev/null +++ b/src/domain/broker.go @@ -0,0 +1,80 @@ +package domain + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + StreamName = "attachment.jobs" + ConsumerGroup = "workers" +) + +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` +} + +type Broker struct{ rdb *redis.Client } + +func NewBroker(ctx context.Context, url string) (*Broker, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(ctx).Err(); err != nil { + return nil, err + } + // Группа создаётся идемпотентно. + err = rdb.XGroupCreateMkStream(ctx, StreamName, ConsumerGroup, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return nil, err + } + return &Broker{rdb: rdb}, nil +} + +func (b *Broker) Publish(ctx context.Context, m JobMessage) (string, error) { + buf, err := json.Marshal(m) + if err != nil { + return "", err + } + return b.rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: StreamName, + Values: map[string]any{"payload": buf}, + }).Result() +} + +// Consume — блокирующее чтение из consumer group; ack делает вызывающий. +func (b *Broker) Consume(ctx context.Context, consumer string) ([]redis.XMessage, error) { + res, err := b.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: ConsumerGroup, Consumer: consumer, + Streams: []string{StreamName, ">"}, + Count: 1, Block: 5 * time.Second, + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, nil + } + return res[0].Messages, nil +} + +func (b *Broker) Ack(ctx context.Context, id string) error { + return b.rdb.XAck(ctx, StreamName, ConsumerGroup, id).Err() +} diff --git a/src/domain/cache.go b/src/domain/cache.go new file mode 100644 index 00000000..45032295 --- /dev/null +++ b/src/domain/cache.go @@ -0,0 +1,91 @@ +package domain + +import ( + "context" + "encoding/json" + "time" + + "github.com/redis/go-redis/v9" +) + +// SessionCache — необязательный кэш сессий поверх Redis. +// Источник истины — PostgreSQL. Кэш ускоряет проверку токена. +type SessionCache struct { + rdb *redis.Client +} + +func NewSessionCache(ctx context.Context, url string) (*SessionCache, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + // Жёсткие короткие таймауты: Redis — кэш, а не критический путь. + opt.DialTimeout = 1 * time.Second + opt.ReadTimeout = 500 * time.Millisecond + opt.WriteTimeout = 500 * time.Millisecond + opt.MaxRetries = 0 // без автоматических ретраев: одна попытка — и уходим в БД + opt.PoolTimeout = 1 * time.Second + + rdb := redis.NewClient(opt) + + // Проверка связи при старте — короткая. + c, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if err := rdb.Ping(c).Err(); err != nil { + _ = rdb.Close() + return nil, err + } + return &SessionCache{rdb: rdb}, nil +} + +func (c *SessionCache) userKey(userID string) string { return "user:" + userID } + +// opCtx возвращает контекст с коротким таймаутом для каждой операции. +// Не наследуем HTTP-контекст напрямую, чтобы не блокировать ответ клиенту. +func (c *SessionCache) opCtx() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), 500*time.Millisecond) +} + +// SetUser кладёт публичные данные пользователя в кэш с TTL до конца сессии. +func (c *SessionCache) SetUser(ctx context.Context, userID string, u User, expires time.Time) { + ttl := time.Until(expires) + if ttl <= 0 { + return + } + b, err := json.Marshal(u) + if err != nil { + return + } + opCtx, cancel := c.opCtx() + defer cancel() + _ = c.rdb.Set(opCtx, c.userKey(userID), b, ttl).Err() +} + +func (c *SessionCache) GetUser(ctx context.Context, userID string) (User, bool) { + opCtx, cancel := c.opCtx() + defer cancel() + b, err := c.rdb.Get(opCtx, c.userKey(userID)).Bytes() + if err != nil { + return User{}, false + } + var u User + if err := json.Unmarshal(b, &u); err != nil { + return User{}, false + } + return u, true +} + +// DelUser — на случай удаления пользователя (в lab2 не вызывается). +func (c *SessionCache) DelUser(ctx context.Context, userID string) error { + opCtx, cancel := c.opCtx() + defer cancel() + return c.rdb.Del(opCtx, c.userKey(userID)).Err() +} + +func (c *SessionCache) Ping(ctx context.Context) error { + c2, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + return c.rdb.Ping(c2).Err() +} + +func (c *SessionCache) Close() error { return c.rdb.Close() } diff --git a/src/domain/idempotency.go b/src/domain/idempotency.go new file mode 100644 index 00000000..f340f77d --- /dev/null +++ b/src/domain/idempotency.go @@ -0,0 +1,75 @@ +package domain + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// IdempotencyRecord — сохранённый результат POST. +type IdempotencyRecord struct { + StatusCode int + Response json.RawMessage +} + +// LookupIdempotency возвращает запись, если ключ уже использован. +// Второй аргумент — совпадает ли хеш тела с сохранённым. +func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) { + var ( + storedHash string + rec IdempotencyRecord + ) + err := s.pool.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`, + userID, method, path, key, + ).Scan(&storedHash, &rec.StatusCode, &rec.Response) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, classifyDBError(err) + } + if storedHash != bodyHash { + return nil, true, ErrConflict + } + return &rec, true, nil +} + +// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный +// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency. +func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error { + _, err := s.pool.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1,$2,$3,$4,$5,$6,$7)`, + userID, method, path, key, bodyHash, status, resp) + return classifyDBError(err) +} + +// HashBody — стабильный хеш JSON независимо от порядка ключей. +func HashBody(raw []byte) (string, error) { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return "", err + } + normalized, err := json.Marshal(v) // map keys сортируются + if err != nil { + return "", err + } + sum := sha256.Sum256(normalized) + return hex.EncodeToString(sum[:]), nil +} + +// CleanupIdempotency — можно звать из фонового тикера (опционально). +func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error { + _, err := s.pool.Exec(ctx, + `DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`, + olderThan.String()) + return err +} diff --git a/src/domain/jobs.go b/src/domain/jobs.go new file mode 100644 index 00000000..20da748d --- /dev/null +++ b/src/domain/jobs.go @@ -0,0 +1,13 @@ +package domain + +import "time" + +type Job struct { + JobID string + AttachmentID string + PipelineVersion int + Attempt int + Status string + NextAttemptAt time.Time + LastError *string +} diff --git a/src/domain/models.go b/src/domain/models.go new file mode 100644 index 00000000..26be699d --- /dev/null +++ b/src/domain/models.go @@ -0,0 +1,94 @@ +package domain + +import ( + "errors" + "time" +) + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt time.Time `json:"created_at"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type MessagePage struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ChatPage struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type Session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt time.Time `json:"expires_at"` + User User `json:"user"` +} + +type ErrorBody struct { + Error ErrorDetail `json:"error"` +} + +type ErrorDetail struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + ObjectKey string `json:"-"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + PipelineVersion int `json:"-"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrInvalid = errors.New("invalid request") + ErrUnauthorized = errors.New("unauthorized") + ErrTimeout = errors.New("timeout") + ErrUnavailable = errors.New("unavailable") + ErrTooLarge = errors.New("payload too large") + ErrUnsupported = errors.New("unsupported media type") + ErrNotImage = errors.New("not a valid image") +) diff --git a/src/domain/postgres.go b/src/domain/postgres.go new file mode 100644 index 00000000..a8689d1e --- /dev/null +++ b/src/domain/postgres.go @@ -0,0 +1,619 @@ +package domain + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "encoding/json" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "golang.org/x/crypto/argon2" +) + +type Store struct { + pool *pgxpool.Pool +} + +func NewStore(ctx context.Context, dsn string) (*Store, error) { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, classifyDBError(err) + } + cfg.MaxConns = 10 + cfg.MinConns = 1 + cfg.MaxConnLifetime = time.Hour + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 5 * time.Second + + cfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { + _, err := conn.Exec(ctx, ` + SET statement_timeout = '5s'; + SET lock_timeout = '2s'; + SET idle_in_transaction_session_timeout = '5s'; + `) + return err + } + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, classifyDBError(err) + } + ctxPing, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := pool.Ping(ctxPing); err != nil { + pool.Close() + return nil, classifyDBError(err) + } + return &Store{pool: pool}, nil +} + +func (s *Store) Close() { s.pool.Close() } + +func (s *Store) Ping(ctx context.Context) error { + c, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return s.pool.Ping(c) +} + +// Argon2id параметры — OWASP baseline: m=19456 KiB (19 MiB), t=2, p=1. +// Хеш кодируется в формате PHC: $argon2id$v=19$m=19456,t=2,p=1$$ +const ( + argonMemory uint32 = 19456 + argonTime uint32 = 2 + argonThreads uint8 = 1 + argonKeyLen uint32 = 32 + argonSaltLen = 16 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + key := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + b64Salt := base64.RawStdEncoding.EncodeToString(salt) + b64Key := base64.RawStdEncoding.EncodeToString(key) + // PHC string; параметры можно будет менять в будущем, старые хеши останутся валидными. + return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$%s$%s", + argonMemory, argonTime, argonThreads, b64Salt, b64Key), nil +} + +func verifyPassword(password, encoded string) bool { + // Минимальный парсер PHC-строки. + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false + } + var mem, time uint32 + var threads uint8 + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &time, &threads); err != nil { + return false + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false + } + wantKey, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false + } + gotKey := argon2.IDKey([]byte(password), salt, time, mem, threads, uint32(len(wantKey))) + return subtle.ConstantTimeCompare(gotKey, wantKey) == 1 +} + +// --- Users --- + +func (s *Store) CreateUser(ctx context.Context, username, displayName, password string) (*User, error) { + hash, err := hashPassword(password) + if err != nil { + return nil, classifyDBError(err) + } + id := uuid.NewString() + var u User + err = s.pool.QueryRow(ctx, + `INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1,$2,$3,$4) + RETURNING id, username, display_name, created_at`, + id, username, displayName, string(hash), + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return nil, ErrConflict + } + return nil, classifyDBError(err) + } + return &u, nil +} + +func (s *Store) GetUserByID(ctx context.Context, id string) (*User, error) { + var u User + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at FROM users WHERE id=$1`, id, + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + return &u, nil +} + +// VerifyPassword возвращает пользователя, если пароль верный; иначе ErrUnauthorized. +func (s *Store) VerifyPassword(ctx context.Context, username, password string) (*User, error) { + var u User + var hash string + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at, password_hash FROM users WHERE username=$1`, + username, + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt, &hash) + if errors.Is(err, pgx.ErrNoRows) { + // Тратим сопоставимое время на проверку фиктивного хеша, + // чтобы не различать "нет пользователя" и "неверный пароль" по таймингу. + _ = verifyPassword(password, "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + return nil, ErrUnauthorized + } + if err != nil { + return nil, classifyDBError(err) + } + if !verifyPassword(password, hash) { + return nil, ErrUnauthorized + } + return &u, nil +} + +// --- Chats --- + +func (s *Store) CreateChat(ctx context.Context, title string, memberIDs []string, creatorID string) (*Chat, error) { + // Проверка входных данных. + seen := map[string]bool{} + for _, id := range memberIDs { + if seen[id] { + return nil, ErrInvalid + } + seen[id] = true + } + if !seen[creatorID] { + return nil, ErrInvalid + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, classifyDBError(err) + } + defer tx.Rollback(ctx) + + // Проверяем, что все участники существуют. + for id := range seen { + var exists bool + if err := tx.QueryRow(ctx, `SELECT true FROM users WHERE id=$1`, id).Scan(&exists); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrInvalid + } + return nil, classifyDBError(err) + } + } + + chatID := uuid.NewString() + var c Chat + err = tx.QueryRow(ctx, + `INSERT INTO chats (id, title) VALUES ($1,$2) RETURNING id, title, created_at`, + chatID, title, + ).Scan(&c.ID, &c.Title, &c.CreatedAt) + if err != nil { + return nil, classifyDBError(err) + } + + for id := range seen { + if _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2)`, + c.ID, id, + ); err != nil { + return nil, classifyDBError(err) + } + } + + if err := tx.Commit(ctx); err != nil { + return nil, classifyDBError(err) + } + + // Порядок member_ids — как в запросе. + c.MemberIDs = append([]string(nil), memberIDs...) + return &c, nil +} + +func (s *Store) GetChat(ctx context.Context, chatID, userID string) (*Chat, error) { + var exists bool + err := s.pool.QueryRow(ctx, + `SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`, + chatID, userID, + ).Scan(&exists) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + + var c Chat + err = s.pool.QueryRow(ctx, + `SELECT id, title, created_at FROM chats WHERE id=$1`, chatID, + ).Scan(&c.ID, &c.Title, &c.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, classifyDBError(err) + } + + rows, err := s.pool.Query(ctx, + `SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chatID) + if err != nil { + return nil, classifyDBError(err) + } + defer rows.Close() + for rows.Next() { + var uid string + if err := rows.Scan(&uid); err != nil { + return nil, classifyDBError(err) + } + c.MemberIDs = append(c.MemberIDs, uid) + } + return &c, rows.Err() +} + +// ListChats возвращает чаты пользователя с keyset-пагинацией по (created_at, id). +func (s *Store) ListChats(ctx context.Context, userID string, limit int, cursor string) ([]Chat, *string, error) { + var rows pgx.Rows + var err error + + if cursor == "" { + rows, err = s.pool.Query(ctx, + `SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = $1 + ORDER BY c.created_at, c.id + LIMIT $2`, userID, limit+1) + } else { + var cCreated time.Time + if err := s.pool.QueryRow(ctx, + `SELECT created_at FROM chats WHERE id=$1`, cursor, + ).Scan(&cCreated); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil, ErrInvalid + } + return nil, nil, classifyDBError(err) + } + rows, err = s.pool.Query(ctx, + `SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = $1 + AND (c.created_at, c.id) > ($2, $3) + ORDER BY c.created_at, c.id + LIMIT $4`, userID, cCreated, cursor, limit+1) + } + if err != nil { + return nil, nil, classifyDBError(err) + } + defer rows.Close() + + var chats []Chat + for rows.Next() { + var c Chat + if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil { + return nil, nil, classifyDBError(err) + } + chats = append(chats, c) + } + if err := rows.Err(); err != nil { + return nil, nil, classifyDBError(err) + } + + var next *string + if len(chats) > limit { + chats = chats[:limit] + last := chats[len(chats)-1].ID + next = &last + } + if chats == nil { + chats = []Chat{} + } + + // Догружаем участников. + for i := range chats { + mrows, err := s.pool.Query(ctx, + `SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chats[i].ID) + if err != nil { + return nil, nil, classifyDBError(err) + } + for mrows.Next() { + var uid string + if err := mrows.Scan(&uid); err != nil { + mrows.Close() + return nil, nil, classifyDBError(err) + } + chats[i].MemberIDs = append(chats[i].MemberIDs, uid) + } + mrows.Close() + } + return chats, next, nil +} + +// --- Messages --- + +func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (*Message, error) { + // Проверка членства + вставка в одной транзакции. + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, classifyDBError(err) + } + defer tx.Rollback(ctx) + + var ok bool + if err := tx.QueryRow(ctx, + `SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`, + chatID, senderID, + ).Scan(&ok); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, classifyDBError(err) + } + + var m Message + var attach []string + err = tx.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) + VALUES ($1,$2,$3,$4,$5) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`, + uuid.NewString(), chatID, senderID, text, []string{}, + ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach) + if err != nil { + return nil, classifyDBError(err) + } + if err := tx.Commit(ctx); err != nil { + return nil, classifyDBError(err) + } + m.AttachmentIDs = attach + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + return &m, nil +} + +// ListMessages — keyset-пагинация по (created_at, id). +// Курсор — это id последнего элемента предыдущей страницы; сам id уникален, +// поэтому используем его для поиска (created_at, id) в БД. +func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cursor string) ([]Message, *string, error) { + var rows pgx.Rows + var err error + + if cursor == "" { + rows, err = s.pool.Query(ctx, + `SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE chat_id=$1 + ORDER BY created_at, id + LIMIT $2`, chatID, limit+1) + } else { + // Определяем позицию курсора. + var cCreated time.Time + if err := s.pool.QueryRow(ctx, + `SELECT created_at FROM messages WHERE id=$1 AND chat_id=$2`, + cursor, chatID, + ).Scan(&cCreated); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil, ErrInvalid + } + return nil, nil, classifyDBError(err) + } + rows, err = s.pool.Query(ctx, + `SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE chat_id=$1 + AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4`, chatID, cCreated, cursor, limit+1) + } + if err != nil { + return nil, nil, classifyDBError(err) + } + defer rows.Close() + + var items []Message + for rows.Next() { + var m Message + if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil { + return nil, nil, classifyDBError(err) + } + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + items = append(items, m) + } + if err := rows.Err(); err != nil { + return nil, nil, classifyDBError(err) + } + + var next *string + if len(items) > limit { + items = items[:limit] + last := items[len(items)-1].ID + next = &last + } + if items == nil { + items = []Message{} + } + return items, next, nil +} + +func classifyDBError(err error) error { + if err == nil { + return nil + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "57014": // query_canceled (statement_timeout) + return ErrTimeout + case "55P03": // lock_not_available (lock_timeout) + return ErrTimeout + case "40001": // serialization_failure + return ErrUnavailable + case "53300": // too_many_connections + return ErrUnavailable + case "22P02", "22007", "22023": // invalid_text_representation / invalid_datetime_format / invalid_parameter_value + return ErrInvalid + } + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return ErrTimeout + } + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + return err +} + +// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение +// и сохраняет результат идемпотентности в одной транзакции. +// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true. +func (s *Store) AddMessageIdempotent( + ctx context.Context, + chatID, senderID, text string, + attachmentIDs []string, + key, bodyHash string, +) (msg *Message, rec *IdempotencyRecord, replay bool, err error) { + path := "/api/v1/chats/" + chatID + "/messages" + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + return nil, nil, false, classifyDBError(err) + } + defer tx.Rollback(ctx) + + // 1) Членство. + var ok bool + if err := tx.QueryRow(ctx, + `SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`, + chatID, senderID, + ).Scan(&ok); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil, false, ErrNotFound + } + return nil, nil, false, classifyDBError(err) + } + + // Вложения должны принадлежать этому чату и быть ready. + if len(attachmentIDs) > 10 { + return nil, nil, false, ErrInvalid + } + seen := map[string]bool{} + for _, aid := range attachmentIDs { + if seen[aid] { + return nil, nil, false, ErrInvalid + } + seen[aid] = true + var ( + status string + objectKey string + ) + // Вложение из этого же чата; не обязательно ready — клиент увидит картинку, + // когда worker завершит обработку. failed — принимаем: сообщение уже создано, + // UI покажет «failed» через GET /attachments/{id}. + if err := tx.QueryRow(ctx, + `SELECT status, object_key FROM attachments WHERE id=$1 AND chat_id=$2`, + aid, chatID).Scan(&status, &objectKey); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil, false, ErrNotFound + } + return nil, nil, false, classifyDBError(err) + } + if status == AttachmentFailed { + // Нельзя прикреплять заведомо неудачные вложения — иначе UI будет + // навсегда показывать сломанную картинку без пути к replay. + return nil, nil, false, ErrInvalid + } + } + + // 2) Уже был такой ключ? + var storedHash string + var storedStatus int + var storedResp json.RawMessage + err = tx.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`, + senderID, path, key, + ).Scan(&storedHash, &storedStatus, &storedResp) + switch { + case err == nil: + if storedHash != bodyHash { + return nil, nil, false, ErrConflict + } + return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil + case !errors.Is(err, pgx.ErrNoRows): + return nil, nil, false, classifyDBError(err) + } + + // 3) Вставка сообщения. + var m Message + var attach []string + err = tx.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) + VALUES ($1,$2,$3,$4,$5) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`, + uuid.NewString(), chatID, senderID, text, attachmentIDs, + ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach) + if err != nil { + // Гонка на одном ключе в разных репликах: unique violation в idempotency_keys + // маловероятен здесь, но serialization failure возможен. + return nil, nil, false, classifyDBError(err) + } + m.AttachmentIDs = attach + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + + // 4) Сохранение ответа. + respJSON, _ := json.Marshal(m) + _, err = tx.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1,'POST',$2,$3,$4,201,$5)`, + senderID, path, key, bodyHash, respJSON) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + // Кто-то параллельно успел вставить: перечитываем в этой же tx. + if err2 := tx.QueryRow(ctx, + `SELECT request_hash, status_code, response + FROM idempotency_keys + WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`, + senderID, path, key, + ).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil { + if storedHash != bodyHash { + return nil, nil, false, ErrConflict + } + return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil + } + } + return nil, nil, false, classifyDBError(err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, nil, false, classifyDBError(err) + } + return &m, nil, false, nil +} diff --git a/src/domain/s3.go b/src/domain/s3.go new file mode 100644 index 00000000..dff225ed --- /dev/null +++ b/src/domain/s3.go @@ -0,0 +1,258 @@ +package domain + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type S3Store struct { + client *minio.Client + bucket string + maxBytes int64 + maxPixels int64 + tr *http.Transport +} + +func NewS3Store(ctx context.Context, endpoint, region, bucket, access, secret string, maxBytes, maxPixels int64) (*S3Store, error) { + if endpoint == "" || access == "" || secret == "" || bucket == "" { + return nil, errors.New("s3 configuration incomplete") + } + + tr := &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 30 * time.Second, + DialContext: (&net.Dialer{ + Timeout: 3 * time.Second, // Быстрый отказ при недоступности + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 3 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + } + + cli, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(access, secret, ""), + Secure: strings.HasPrefix(endpoint, "https://"), + Region: region, + Transport: tr, + }) + if err != nil { + return nil, err + } + + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if _, err := cli.BucketExists(pingCtx, bucket); err != nil { + return nil, fmt.Errorf("s3 bucket check: %w", err) + } + return &S3Store{client: cli, bucket: bucket, maxBytes: maxBytes, maxPixels: maxPixels, tr: tr}, nil +} + +type UploadResult struct { + ObjectKey string + ContentType string + SizeBytes int64 + SHA256 string + Width int + Height int +} + +// PutValidatedImage читает тело потоково во временный файл на диске, +// проверяет размер, MIME, пиксели и затем стримит в S3. +// Это гарантирует, что память не растёт с размером файла. +func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body io.Reader) (*UploadResult, error) { + // 1. Создаём временный файл. + tmpFile, err := os.CreateTemp("", "upload-*") + if err != nil { + return nil, err + } + tmpPath := tmpFile.Name() + defer func() { + tmpFile.Close() + os.Remove(tmpPath) + }() + + // 2. Копируем тело в файл с ограничением. + limited := io.LimitReader(body, s.maxBytes+1) + size, err := io.Copy(tmpFile, limited) + if err != nil { + return nil, err + } + if size > s.maxBytes { + return nil, ErrTooLarge + } + if size == 0 { + return nil, ErrUnsupported + } + + // 3. Проверяем MIME по первым 512 байтам. + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return nil, err + } + header := make([]byte, 512) + n, err := io.ReadFull(tmpFile, header) + if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { + return nil, err + } + header = header[:n] + ct := http.DetectContentType(header) + if ct != "image/png" && ct != "image/jpeg" { + return nil, ErrUnsupported + } + + // 4. Проверяем заголовок изображения (размеры). + // DecodeConfig читает только начало, но мы используем файл, поэтому это безопасно. + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return nil, err + } + cfg, _, err := image.DecodeConfig(tmpFile) + if err != nil { + return nil, ErrNotImage + } + if int64(cfg.Width)*int64(cfg.Height) > s.maxPixels { + return nil, ErrNotImage + } + + // 5. Считаем SHA-256 и стримим файл в S3. + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return nil, err + } + var hasher hash.Hash = sha256.New() + tee := io.TeeReader(tmpFile, hasher) + + putCtx, cancelPut := context.WithTimeout(ctx, 5*time.Second) + defer cancelPut() + + _, err = s.client.PutObject(putCtx, s.bucket, objectKey, tee, size, + minio.PutObjectOptions{ContentType: ct}) + if err != nil { + return nil, err + } + + sha := hex.EncodeToString(hasher.Sum(nil)) + + return &UploadResult{ + ObjectKey: objectKey, + ContentType: ct, + SizeBytes: size, + SHA256: sha, + Width: cfg.Width, + Height: cfg.Height, + }, nil +} + +const ( + StagingPrefix = "staging/" + FinalPrefix = "final/" +) + +// MoveToFinal: копирование staging→final и удаление staging. +// Используем CopyObject, чтобы не тянуть байты в память. Если MinIO не умеет +// server-side copy — используйте compose? MinIO умеет S3 CopyObject. +func (s *S3Store) MoveToFinal(ctx context.Context, stagingKey, finalKey string) error { + src := minio.CopySrcOptions{Bucket: s.bucket, Object: stagingKey} + dst := minio.CopyDestOptions{Bucket: s.bucket, Object: finalKey} + if _, err := s.client.CopyObject(ctx, dst, src); err != nil { + return err + } + return s.DeleteObject(ctx, stagingKey) +} + +func (s *S3Store) PutBytes(ctx context.Context, key, contentType string, data []byte) (string, error) { + h := sha256.Sum256(data) + _, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: contentType}) + if err != nil { + return "", err + } + return hex.EncodeToString(h[:]), nil +} + +// FetchStagingForProcessing — читает staging в память с жёстким лимитом. +// maxBytes гарантирует, что worker не съест RAM на «злом» файле. +func (s *S3Store) FetchStagingForProcessing(ctx context.Context, key string) ([]byte, string, error) { + rc, ct, size, err := s.GetObject(ctx, key) + if err != nil { + return nil, "", err + } + defer rc.Close() + if size > s.maxBytes { + return nil, "", ErrTooLarge + } + buf, err := io.ReadAll(rc) + if err != nil { + return nil, "", err + } + return buf, ct, nil +} + +func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) { + // Общий дедлайн на все попытки — 6 секунд. + overallCtx, overallCancel := context.WithTimeout(ctx, 6*time.Second) + defer overallCancel() + + var lastErr error + for attempt := 0; attempt < 2; attempt++ { + if attempt > 0 { + time.Sleep(500 * time.Millisecond) + s.tr.CloseIdleConnections() + } + + attemptCtx, cancel := context.WithTimeout(overallCtx, 3*time.Second) + obj, err := s.client.GetObject(attemptCtx, s.bucket, objectKey, minio.GetObjectOptions{}) + if err != nil { + cancel() + lastErr = err + continue + } + + info, err := obj.Stat() + if err != nil { + obj.Close() + cancel() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", 0, ErrNotFound + } + lastErr = err + continue + } + + data, err := io.ReadAll(obj) + obj.Close() + cancel() + if err != nil { + lastErr = err + continue + } + if int64(len(data)) != info.Size { + lastErr = fmt.Errorf("size mismatch: got %d, expected %d", len(data), info.Size) + continue + } + + return io.NopCloser(bytes.NewReader(data)), info.ContentType, info.Size, nil + } + + s.tr.CloseIdleConnections() + return nil, "", 0, fmt.Errorf("s3 get failed after retries: %w", lastErr) +} + +func (s *S3Store) DeleteObject(ctx context.Context, objectKey string) error { + return s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{}) +} diff --git a/src/domain/sessions.go b/src/domain/sessions.go new file mode 100644 index 00000000..5eaf1786 --- /dev/null +++ b/src/domain/sessions.go @@ -0,0 +1,108 @@ +package domain + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +type SessionStore struct { + store *Store + cache *SessionCache + ttl time.Duration +} + +func NewSessionStore(store *Store, cache *SessionCache, ttl time.Duration) *SessionStore { + return &SessionStore{store: store, cache: cache, ttl: ttl} +} + +func newToken() (string, error) { + b := make([]byte, 32) // 256 бит + if _, err := rand.Read(b); err != nil { + return "", classifyDBError(err) + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// Create — пишем сессию в БД (источник истины) и кэшируем публичные данные User. +func (s *SessionStore) Create(ctx context.Context, user User) (*Session, error) { + token, err := newToken() + if err != nil { + return nil, classifyDBError(err) + } + expires := time.Now().UTC().Add(s.ttl) + if _, err := s.store.pool.Exec(ctx, + `INSERT INTO sessions (token, user_id, expires_at) VALUES ($1,$2,$3)`, + token, user.ID, expires); err != nil { + return nil, classifyDBError(err) + } + if s.cache != nil { + s.cache.SetUser(ctx, user.ID, user, expires) + } + return &Session{ + Token: token, + TokenType: "Bearer", + ExpiresAt: expires, + User: user, + }, nil +} + +// Lookup ВСЕГДА обращается к PostgreSQL за фактом существования сессии. +// Кэш используется только для публичных данных User. +func (s *SessionStore) Lookup(ctx context.Context, token string) (*User, error) { + var ( + userID string + expiresAt time.Time + ) + err := s.store.pool.QueryRow(ctx, + `SELECT user_id, expires_at FROM sessions WHERE token=$1`, token, + ).Scan(&userID, &expiresAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrUnauthorized + } + if err != nil { + return nil, classifyDBError(err) + } + if time.Now().UTC().After(expiresAt) { + _, _ = s.store.pool.Exec(ctx, `DELETE FROM sessions WHERE token=$1`, token) + return nil, ErrUnauthorized + } + + if s.cache != nil { + if u, ok := s.cache.GetUser(ctx, userID); ok { + return &u, nil + } + } + + u, err := s.store.GetUserByID(ctx, userID) + if err != nil { + return nil, ErrUnauthorized + } + if s.cache != nil { + s.cache.SetUser(ctx, userID, *u, expiresAt) + } + return u, nil +} + +func (s *SessionStore) Revoke(ctx context.Context, token string) error { + var userID string + err := s.store.pool.QueryRow(ctx, + `DELETE FROM sessions WHERE token=$1 RETURNING user_id`, token, + ).Scan(&userID) + if errors.Is(err, pgx.ErrNoRows) { + // Сессии уже нет — с точки зрения клиента токен отозван. + return nil + } + if err != nil { + return classifyDBError(err) + } + if s.cache != nil { + // Кэш User не критичен, но полезно сбросить. + _ = s.cache.DelUser(ctx, userID) + } + return nil +} diff --git a/student_tests/broker-payload.py b/student_tests/broker-payload.py new file mode 100644 index 00000000..d552066f --- /dev/null +++ b/student_tests/broker-payload.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""В Stream нет filename/URL/секретов — только id/version/attempt.""" +import json, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + upload(c, a, ch, content) + raw = docker('exec', '-T', 'redis', 'redis-cli', '--raw', 'XRANGE', 'attachment.jobs', '-', '+') + ok('stream sample:\n' + raw) + lower = raw.lower() + for bad in ('filename', 'sample.png', 'staging/', 'final/', 'http', 'secret'): + check(bad not in lower, 'broker payload contains forbidden token: %s' % bad) + ok('broker payload has only ids/version/attempt') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 00000000..4732d3f2 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,73 @@ +"""Shared helpers for student lab5 tests; no tokens are logged.""" +import base64, json, os, subprocess, sys, time, uuid +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, Failure, check # noqa: E402 + +def fail(msg): + print('FAIL:', msg) + raise SystemExit(1) + +def ok(msg='PASS'): + print(msg) + +def docker(*args, check_rc=True): + """Run docker compose ; return stdout. Raises if rc!=0 and check_rc.""" + rc = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True) + if check_rc and rc.returncode != 0: + raise Failure('docker compose %s: %s' % (' '.join(args), rc.stderr.strip())) + return rc.stdout + +def psql(sql): + return docker('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-tAc', sql).strip() + +def upload(c, actor, chat, content, filename='sample.png', ct='image/png'): + return c.call('POST', + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, + 202, actor=actor, raw=content, headers={'Content-Type': ct}, + schema='Attachment')['json'] + +def wait_status(c, actor, att_id, want, seconds=60): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] == want: + return att + if att['status'] == 'failed' and want != 'failed': + fail('attachment failed: ' + str(att.get('error_code'))) + time.sleep(0.25) + fail('attachment did not reach %s in %ss' % (want, seconds)) + +def png_bytes(w, h, seed=0): + """Generate a valid PNG without external libs. + + Быстро и с малой памятью: одна строка пикселей переиспользуется. + Для 7000x7000 занимает ~1-2 секунды и ~150 MiB RAM. + """ + import struct, zlib + + # одна строка: filter byte 0 + w * 3 байта RGB + row = bytearray(1 + w * 3) + for x in range(w): + row[1 + x*3] = (x * 7 + seed) & 0xFF + row[2 + x*3] = (x * 11) & 0xFF + row[3 + x*3] = (x * 13) & 0xFF + row = bytes(row) + + # zlib потоково, без накопления всего raw + co = zlib.compressobj(level=6) + idat_parts = [] + for y in range(h): + idat_parts.append(co.compress(row)) + idat_parts.append(co.flush()) + idat = b''.join(idat_parts) + + def chunk(t, d): + return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff) + + ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', ihdr) + + chunk(b'IDAT', idat) + + chunk(b'IEND', b'')) \ No newline at end of file diff --git a/student_tests/crash-before-ack.py b/student_tests/crash-before-ack.py new file mode 100644 index 00000000..76e8da65 --- /dev/null +++ b/student_tests/crash-before-ack.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""kill -9 worker после S3 write до ack → повтор не создаёт дублей.""" +import sys, time, signal +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = png_bytes(4000, 4000) + + # Сначала готовим staging+job, но без запуска worker’а. + docker('stop', 'worker') + att = upload(c, a, ch, content) + # Запускаем worker — он возьмёт processing и, скорее всего, быстро закончит. + docker('start', 'worker') + # Ловим момент processing и убиваем процесс. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + if st == 'processing': + break + time.sleep(0.1) + docker('kill', '-s', 'SIGKILL', 'worker') + ok('worker killed during processing') + + # После lease (60s) + reaper (30s) job должна снова стать queued. + time.sleep(100) + docker('start', 'worker') + att = wait_status(c, a, att['id'], 'ready', seconds=120) + n = int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])) + check(n == 3, 'expected exactly 3 variants, got %d' % n) + ok('recovery produced exactly 3 non-original variants, no duplicates (original in object_key)') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/dlq.py b/student_tests/dlq.py new file mode 100644 index 00000000..1e90520b --- /dev/null +++ b/student_tests/dlq.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Реестр failed доступен оператору (SELECT из attachment_jobs).""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes()[:-20] + att = upload(c, a, ch, content) + wait_status(c, a, att['id'], 'failed', seconds=180) + rows = psql( + "SELECT attachment_id, last_error_code, attempt FROM attachment_jobs " + "WHERE status='failed' ORDER BY updated_at DESC LIMIT 5") + check(rows, 'no failed jobs visible to operator') + ok('failed registry:\n' + rows) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/failed.py b/student_tests/failed.py new file mode 100644 index 00000000..47aa8daa --- /dev/null +++ b/student_tests/failed.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Битый файл: 5 attempt → failed, error_code=invalid_image.""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + # Заведомо валидный PNG в API (MIME проходит), но worker получит + # битый заголовок, если подменить staging — здесь проверяем + # реалистичный сценарий: JPEG с обрезанным хвостом. + content = (Path('tests/fixtures/sample.png')).read_bytes()[:-20] + att = upload(c, a, ch, content) + # API пропустит (DecodeConfig проходит), worker упадёт на Decode. + att = wait_status(c, a, att['id'], 'failed', seconds=180) + check(att.get('error_code') == 'invalid_image', 'unexpected error_code: %s' % att.get('error_code')) + attempts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])) + # invalid_image — постоянная ошибка, worker не должен ретраить. + check(attempts == 1, 'permanent error must not be retried: attempt=%d' % attempts) + ok('failed immediately (attempt=1), error_code=invalid_image — permanent, not retried') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/jpeg_probe.py b/student_tests/jpeg_probe.py new file mode 100644 index 00000000..1ea8051f --- /dev/null +++ b/student_tests/jpeg_probe.py @@ -0,0 +1,28 @@ +"""Мини-парсер SOF для JPEG; без внешних библиотек.""" +from tests.client import Failure, check + + +def jpeg_size(data): + check(data[:2] == b'\xff\xd8', 'thumbnail is not JPEG') + i = 2 + sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf} + while i < len(data): + check(data[i] == 0xff, 'invalid JPEG marker') + while i < len(data) and data[i] == 0xff: + i += 1 + check(i < len(data), 'truncated JPEG marker') + marker = data[i] + i += 1 + if marker in {0xd9, 0xda}: + break + if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7: + continue + check(i + 2 <= len(data), 'truncated JPEG segment') + length = int.from_bytes(data[i:i + 2], 'big') + check(length >= 2 and i + length <= len(data), 'invalid JPEG segment length') + if marker in sof: + check(length >= 8, 'invalid JPEG SOF') + return (int.from_bytes(data[i + 5:i + 7], 'big'), + int.from_bytes(data[i + 3:i + 5], 'big')) + i += length + raise Failure('JPEG dimensions not found') \ No newline at end of file diff --git a/student_tests/lease.py b/student_tests/lease.py new file mode 100644 index 00000000..e77c5c50 --- /dev/null +++ b/student_tests/lease.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Reaper восстанавливает job после паузы worker’а дольше lease (60с).""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = png_bytes(4000, 4000) + + # 1) Готовим job, но worker не даём её взять. + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + finally: + pass # unpause ниже — после того, как настроим SIGSTOP + + # 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP. + # Используем docker kill --signal=SIGSTOP, чтобы остановить процесс + # (не контейнер). Контейнер "жив", но worker заморожен. + docker('unpause', 'worker') + + # 3) Ждём processing; как только взял — сразу SIGSTOP. + deadline = time.monotonic() + 15 + job = None + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + if st == 'processing': + # Останавливаем процесс, пока он внутри processing. + docker('kill', '-s', 'SIGSTOP', 'worker') + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + break + if st == 'done': + fail('job finished too fast to simulate stuck worker') + time.sleep(0.02) + + check(job is not None, 'worker never entered processing') + + # 4) Фиксируем lease_expires_at на момент SIGSTOP. + lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job) + ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease)) + + # 5) Ждём, пока lease протухнет и reaper вернёт job в queued. + # lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом. + deadline = time.monotonic() + 120 + requeued = False + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job) + if st == 'queued': + requeued = True + break + time.sleep(2) + check(requeued, 'reaper did not requeue stuck job (status=%s)' % st) + ok('reaper requeued the stuck job') + + # 6) Возобновляем worker: он должен доделать job до ready. + docker('kill', '-s', 'SIGCONT', 'worker') + att = wait_status(c, a, att['id'], 'ready', seconds=120) + ok('ready after unstuck') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/limits.py b/student_tests/limits.py new file mode 100644 index 00000000..9a0d6180 --- /dev/null +++ b/student_tests/limits.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""413, 415, 400 invalid_image — лимиты bytes/pixels/MIME.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import png_bytes, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + url = '/api/v1/chats/' + ch['id'] + '/attachments?filename=x.bin' + + # 413: > 10 MiB + big = b'\x89PNG\r\n\x1a\n' + b'\x00' * (11 * 1024 * 1024) + r = c.call('POST', url, 413, actor=a, raw=big, headers={'Content-Type': 'image/png'}) + ok('413 payload_too_large') + + # 415: text/plain + r = c.call('POST', url, 415, actor=a, raw=b'hello', headers={'Content-Type': 'text/plain'}) + ok('415 unsupported_media_type') + + # 415: клиент говорит image/png, но содержимое не PNG/JPEG по DetectContentType + r = c.call('POST', url, 415, actor=a, raw=b'\x00' * 100, headers={'Content-Type': 'image/png'}) + ok('415 mismatched content (client MIME not trusted)') + + # 400: PNG-сигнатура валидна (DetectContentType -> image/png), + # но IHDR/IDAT битые -> DecodeConfig падает -> invalid_image (400) + broken_png = b'\x89PNG\r\n\x1a\n' + b'\x00' * 40 + r = c.call('POST', url, 400, actor=a, raw=broken_png, headers={'Content-Type': 'image/png'}) + ok('400 invalid image (signature ok, header broken)') + + # 400: пиксели > MAX_IMAGE_PIXELS (если выставите 1e8 — этот тест не сработает; + # по умолчанию 40 MP, значит нужно 7000x7000 ≈ 49 MP) + huge = png_bytes(7000, 7000) + r = c.call('POST', url, 400, actor=a, raw=huge, headers={'Content-Type': 'image/png'}) + ok('400 too many pixels') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/outbox.py b/student_tests/outbox.py new file mode 100644 index 00000000..da941c73 --- /dev/null +++ b/student_tests/outbox.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Outbox: пауза outbox → 202 → unpause → ready; published_at=NULL в промежутке.""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + docker('pause', 'outbox') + try: + att = upload(c, a, ch, content) + check(att['status'] == 'queued', 'expected queued') + time.sleep(3) + published = psql("SELECT coalesce(published_at::text,'NULL') FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + check(published == 'NULL', 'outbox paused but published_at=%s' % published) + ok('outbox paused: published_at IS NULL, upload accepted') + finally: + docker('unpause', 'outbox') + + att = wait_status(c, a, att['id'], 'ready', seconds=90) + ok('after unpause: ready') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/redelivery.py b/student_tests/redelivery.py new file mode 100644 index 00000000..416a7c5e --- /dev/null +++ b/student_tests/redelivery.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Повторный XADD того же job_id не создаёт дублей вариантов.""" +import json, subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'ready') + before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']) + check(before == '3', 'expected 3 variants, got %s' % before) + + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + payload = json.dumps({ + "schema_version": 1, "job_id": job, "type": "attachment.process.v1", + "attachment_id": att['id'], "pipeline_version": 1, "attempt": 1, + "created_at": "2026-01-01T00:00:00Z", + }) + # Повторная доставка через Redis CLI. + docker('exec', '-T', 'redis', 'redis-cli', 'XADD', 'attachment.jobs', '*', 'payload', payload) + time.sleep(5) + after = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']) + check(after == '3', 'redelivery created duplicate variants: %s' % after) + ok('redelivery is no-op (still 3 non-original variants; original via object_key)') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/replay.py b/student_tests/replay.py new file mode 100644 index 00000000..e5c20ce7 --- /dev/null +++ b/student_tests/replay.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Replay failed → снова queued.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + # Форсим failed (битый файл проходит DecodeConfig, но не Decode). + content = Path('tests/fixtures/sample.png').read_bytes()[:-20] + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'failed', seconds=180) + ok('failed reached') + + c.call('POST', '/api/v1/attachments/' + att['id'] + '/replay', 202, actor=a) + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + check(st == 'queued', 'after replay expected queued, got %s' % st) + ok('replay -> queued') + + # Replay дал одну новую попытку; ошибка permanent, поэтому снова failed при attempt=1. + att = wait_status(c, a, att['id'], 'failed', seconds=180) + atts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])) + check(atts == 1, 'after replay: permanent error must fail at attempt=1, got %d' % atts) + ok('replay reset attempt to 1; permanent error failed again — not retried') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/retry.py b/student_tests/retry.py new file mode 100644 index 00000000..66ce29e4 --- /dev/null +++ b/student_tests/retry.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready.""" +import subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + # 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем. + root_user = open('secrets/minio_root_user.txt').read().strip() + root_pass = open('secrets/minio_root_password.txt').read().strip() + docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc', + 'http://localhost:9000', root_user, root_pass, '--api', 'S3v4') + + # 2) Пауза worker’а: job не будет взят до unpause. + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id']) + check(staging.startswith('staging/'), 'unexpected staging_key') + + # 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект. + # Если worker ещё не стартовал — GetObject вернёт NoSuchKey. + subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm', + 'loc/messenger-attachments/' + staging], + capture_output=True, text=True, check=True) + + # 4) Отдаём job worker’у. + docker('unpause', 'worker') + + # 5) Ждём, пока attempt >= 2. + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 90 + attempt = 1 + while time.monotonic() < deadline: + attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job)) + if attempt >= 2: + break + time.sleep(1) + check(attempt >= 2, + 'retry did not happen on missing staging object (attempt=%d)' % attempt) + ok('transient error retried: attempt=%d' % attempt) + + # 6) Логи retry именно для нашего job_id. + logs = docker('logs', '--tail=500', 'worker') + lines = [l for l in logs.splitlines() + if ('retry job=' + job) in l] + check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:])) + ok('retry log lines:') + for l in lines: + print(' ' + l) + + # 7) Возвращаем staging — job должен пройти retry и стать ready. + # Перезаписываем объект с правильным содержимым. + # Через mc pipe: передаём байты на stdin. + r = subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', + 'mc', 'pipe', 'loc/messenger-attachments/' + staging], + input=content, capture_output=True) + check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode()) + + att = wait_status(c, a, att['id'], 'ready', seconds=180) + ok('recovered to ready after staging restored (attempt=%d)' % attempt) + + finally: + # На всякий случай — если упали в середине, worker не должен остаться на паузе. + subprocess.run(['docker', 'compose', 'unpause', 'worker'], + capture_output=True) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/staging-cleanup.py b/student_tests/staging-cleanup.py new file mode 100644 index 00000000..9edba4a6 --- /dev/null +++ b/student_tests/staging-cleanup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""После ready staging пуст (объект перенесён в final).""" +import sys +import subprocess +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id']) + check(staging.startswith('staging/'), 'staging_key does not start with staging/') + att = wait_status(c, a, att['id'], 'ready') + root_user = open('secrets/minio_root_user.txt').read().strip() + root_pass = open('secrets/minio_root_password.txt').read().strip() + docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'local_check', + 'http://localhost:9000', root_user, root_pass, '--api', 'S3v4') + + r = subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', 'mc', 'stat', + 'local_check/messenger-attachments/' + staging], + capture_output=True, text=True) + combined = ((r.stdout or '') + (r.stderr or '')).lower() + check('does not exist' in combined or 'not found' in combined, + 'staging object still present: %s\n%s' % (staging, combined)) + ok('staging object removed after ready (mc rc=%d)' % r.returncode) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/states.py b/student_tests/states.py new file mode 100644 index 00000000..ff9ae2c5 --- /dev/null +++ b/student_tests/states.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""queued → processing → ready через остановку worker’а.""" +import time, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + check(att['status'] == 'queued', 'expected queued after upload') + ok('upload -> queued') + # Не должно перейти в processing, пока worker на паузе. + time.sleep(3) + att = c.call('GET', '/api/v1/attachments/' + att['id'], actor=a, schema='Attachment')['json'] + check(att['status'] == 'queued', 'worker paused but status changed to %s' % att['status']) + ok('status stays queued while worker paused') + finally: + docker('unpause', 'worker') + + att = wait_status(c, a, att['id'], 'ready', seconds=90) + names = {v['name'] for v in att['variants']} + # original в variants[] нет по контракту OpenAPI — он в object_key/оригинальном файле. + check(names == {'thumbnail', 'crop', 'optimized'}, + 'variants mismatch: %s' % names) + ok('ready with thumbnail/crop/optimized (original available via ?variant=original)') + + # original доступен отдельно — проверим, что эндпоинт отвечает 200. + r = c.call('GET', + '/api/v1/attachments/' + att['id'] + '/content?variant=original', + 200, actor=a) + check(len(r['bytes']) > 0, 'original empty') + ok('original served via ?variant=original (%d bytes)' % len(r['bytes'])) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/variants.py b/student_tests/variants.py new file mode 100644 index 00000000..13aa2310 --- /dev/null +++ b/student_tests/variants.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""original неизменен, thumbnail 256x256, crop 128x128, optimized существует.""" +import hashlib, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, ok, fail +from student_tests.jpeg_probe import jpeg_size as dims_jpeg + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'ready') + src_sha = hashlib.sha256(content).hexdigest() + + def get(v): + return c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (att['id'], v), + 200, actor=a)['bytes'] + + orig = get('original') + check(hashlib.sha256(orig).hexdigest() == src_sha, 'original changed') + ok('original byte-identical') + + th = get('thumbnail') + w, h = dims_jpeg(th) + check(max(w, h) == 256 and w <= 256 and h <= 256, + 'thumbnail is %dx%d, expected fit in 256x256' % (w, h)) + check(w * 160 == h * 256 if h == 160 else True, ...) # необязательно + # Проще всего — сверить с metadata: + thumb_meta = next(v for v in att['variants'] if v['name'] == 'thumbnail') + check((w, h) == (thumb_meta['width'], thumb_meta['height']), + 'thumbnail metadata %sx%s != actual %dx%d' % (thumb_meta['width'], thumb_meta['height'], w, h)) + ok('thumbnail %dx%d (metadata matches)' % (w, h)) + + cr = get('crop') + w, h = dims_jpeg(cr) + check((w, h) == (128, 128), 'crop is %dx%d' % (w, h)) + ok('crop 128x128 JPEG') + + opt = get('optimized') + check(len(opt) > 0, 'optimized empty') + # optimized может быть JPEG или WebP; проверяем, что не пусто и похоже на изображение + check(opt[:2] == b'\xff\xd8' or opt[:4] == b'RIFF', 'optimized is not JPEG/WebP') + ok('optimized present (%d bytes)' % len(opt)) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file