From d9de5e1a60842f92358942d448bdc74a9e1b43d7 Mon Sep 17 00:00:00 2001 From: Rakshasa2 Date: Thu, 12 Nov 2026 18:48:39 +0300 Subject: [PATCH] =?UTF-8?q?lab5=20-=20=D0=BE=D1=86=D0=B5=D0=BD=D0=BA=D0=B0?= =?UTF-8?q?=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 34 + REPORT.md | 66 +- docker-compose.yml | 198 +++ go.mod | 34 + go.sum | 72 + infra/haproxy.cfg | 30 + src/cmd/server/main.go | 227 +++ src/cmd/worker/main.go | 586 ++++++++ src/internal/auth/auth.go | 46 + src/internal/handler/handler.go | 888 ++++++++++++ src/internal/model/model.go | 79 + src/internal/session/redis_store.go | 47 + src/internal/storage/s3.go | 230 +++ src/internal/store/migrations/001_initial.sql | 47 + .../store/migrations/002_idempotency.sql | 19 + .../store/migrations/003_attachments.sql | 27 + .../store/migrations/004_allow_empty_text.sql | 3 + .../store/migrations/005_async_jobs.sql | 14 + .../006_update_attachment_status.sql | 43 + .../store/migrations/007_async_jobs.sql | 19 + .../migrations/008_attachment_variants.sql | 18 + .../store/migrations/009_variant_contract.sql | 10 + src/internal/store/migrations/010_level4.sql | 27 + .../store/migrations/011_fix_jobs_schema.sql | 59 + .../012_attachment_sha256_index.sql | 5 + src/internal/store/postgres.go | 1273 +++++++++++++++++ student-tests/check-download.py | 57 + student-tests/debug_smoke.py | 60 + student-tests/fault_before_ack.sh | 49 + student-tests/fault_commit_publish.sh | 25 + student-tests/fault_lease.sh | 40 + student-tests/fault_retry_replay.sh | 45 + student-tests/fault_worker_down.sh | 35 + student-tests/lab5_probe.py | 147 ++ student-tests/load_parallel.sh | 10 + student-tests/replay_failed.sh | 29 + student-tests/run-test.py | 60 + 37 files changed, 4639 insertions(+), 19 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/haproxy.cfg create mode 100644 src/cmd/server/main.go create mode 100644 src/cmd/worker/main.go create mode 100644 src/internal/auth/auth.go create mode 100644 src/internal/handler/handler.go create mode 100644 src/internal/model/model.go create mode 100644 src/internal/session/redis_store.go create mode 100644 src/internal/storage/s3.go create mode 100644 src/internal/store/migrations/001_initial.sql create mode 100644 src/internal/store/migrations/002_idempotency.sql create mode 100644 src/internal/store/migrations/003_attachments.sql create mode 100644 src/internal/store/migrations/004_allow_empty_text.sql create mode 100644 src/internal/store/migrations/005_async_jobs.sql create mode 100644 src/internal/store/migrations/006_update_attachment_status.sql create mode 100644 src/internal/store/migrations/007_async_jobs.sql create mode 100644 src/internal/store/migrations/008_attachment_variants.sql create mode 100644 src/internal/store/migrations/009_variant_contract.sql create mode 100644 src/internal/store/migrations/010_level4.sql create mode 100644 src/internal/store/migrations/011_fix_jobs_schema.sql create mode 100644 src/internal/store/migrations/012_attachment_sha256_index.sql create mode 100644 src/internal/store/postgres.go create mode 100644 student-tests/check-download.py create mode 100644 student-tests/debug_smoke.py create mode 100755 student-tests/fault_before_ack.sh create mode 100755 student-tests/fault_commit_publish.sh create mode 100755 student-tests/fault_lease.sh create mode 100755 student-tests/fault_retry_replay.sh create mode 100755 student-tests/fault_worker_down.sh create mode 100755 student-tests/lab5_probe.py create mode 100755 student-tests/load_parallel.sh create mode 100755 student-tests/replay_failed.sh create mode 100644 student-tests/run-test.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b9447f4d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM golang:1.22-alpine AS builder + +WORKDIR /build + +ENV GOPROXY=https://proxy.golang.org,direct +ENV GOSUMDB=sum.golang.org + +COPY go.mod go.sum ./ +RUN go mod download || go mod download || go mod download + +COPY src/ ./src/ + +# Собираем оба бинарника статически +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./src/cmd/server +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/worker ./src/cmd/worker + +FROM alpine:3.20 + +RUN addgroup -S app && adduser -S -G app app \ + && apk add --no-cache ca-certificates tzdata + +WORKDIR /app + +COPY --from=builder /out/server . +COPY --from=builder /out/worker . + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=5 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health/ready || exit 1 + +CMD ["./server"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..ac49978f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,65 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Тюрина Александра, БВТ2304 +- **Целевая оценка:** 4 (трек 5 не заявлен) +- **Commit/PR:** rakshasa/mtusi2026lab5-fork`, ветка `solution/bvt2304-tyurina`. +- **Источники и ИИ:** Go 1.22, chi, pgx, go-redis, minio-go, imaging, nativewebp. ИИ использовался для архитектурного ревью (outbox-эквивалент, lease, идемпотентность эффекта) и отладки. Весь код написан и проверен автором. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +1. `cp .env.example .env` (заполнить пароли, `.env` в `.gitignore`). +2. `docker compose --env-file .env up -d --build` (сборка и запуск, миграции применяются автоматически). +3. `until docker compose ps | grep -q 'lb.*healthy'; do sleep 1; done` (ожидание готовности). +4. `make check && make test` (проверка контракта и публичный smoke). +5. Безопасная остановка: `docker compose --env-file .env down`. +6. Удаление данных: `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** Клиент → HAProxy (2 реплики API) → PostgreSQL (источник истины) + Redis db0 (кэш сессий) + RustFS (S3, байты) + Redis db1 (брокер заданий). Отдельный сервис `worker` обрабатывает очереди. + +**Ключевые решения лабы 5:** +- **Асинхронность:** `POST /attachments` возвращает `202 queued`. Байты сохраняются в `staging/`, метаданные и задание (`attachment_jobs`) пишутся в одной транзакции (outbox-эквивалент). +- **Надёжность обработки:** Worker использует `ClaimJob` с `lease_until` (защита от зависших задач) и `XACK` только после успешного `COMMIT` результата. `reclaim` подхватывает неподтверждённые сообщения из PEL. +- **Retry и DLQ:** Временные ошибки (`s3_*`) ретраятся с backoff (`5с * 2^(n-1)`) и jitter. Постоянные ошибки и исчерпание `max_attempts` переводят задание в `failed` и пишут в DLQ. +- **Идемпотентность эффекта:** Уникальный индекс `(attachment_id, pipeline_version, kind)` гарантирует, что повторная доставка не создаст дубликаты вариантов. +- **Очистка:** Janitor удаляет `staging/` объекты после безопасной фиксации результата. + +**Исправленные дефекты в ходе работы:** +1. Дубль номера миграции `005` (исправлено добавлением `011_fix_jobs_schema.sql`). +2. Инверсия классификации ошибок retry (исправлено на `!perr.pated`). +3. Удалён cross-chat reuse вложений, приводивший к удалению чужих оригиналов (теперь всегда используется уникальный `staging/` ключ). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / Сценарий | Наблюдаемый результат | Файл | +|---|---|---|---| +| **Smoke** | `make test` | `PASS: lab5 public API scenario...` | `evidence/smoke.txt` | +| **Ур.3: 202 + устойчивость** | `fault_worker_down.sh` | `202 queued`, сообщение с вложением принимается (`201`), после рестарта worker → `ready` | `evidence/A_worker_down.txt` | +| **Ур.3: Разрыв COMMIT/publish** | `fault_commit_publish.sh` | Задание `pending` в БД, после подъёма брокера обработано автоматически, без ручного создания | `evidence/B_commit_publish.txt` | +| **Ур.4: Crash до ack** | `fault_before_ack.sh after_commit_before_ack` | Результат `ready` в БД, сообщение в PEL, после рестарта `reclaimed` → `no-op ack`, дублей 0 | `evidence/C_after_commit.txt` | +| **Ур.4: Истёкший lease** | `fault_lease.sh` | Задача `processing` с `lease_until`, после истечения `requeued` и успешно выполнена (attempt=2) | `evidence/D_lease.txt` | +| **Ур.4: Backoff + jitter + DLQ** | `fault_retry_replay.sh` | `retry in 5.01s` (jitter), затем `failed`, запись в DLQ, операторский replay успешен, повторный replay = 0 строк | `evidence/E_retry_replay.txt` | +| **Ур.4: Crop + WebP** | `fault_worker_down.sh` | Оригинал (PNG), thumbnail (JPEG 256x160), crop (JPEG 128x128), optimized (WebP 320x200). SHA совпадают | `evidence/A_worker_down.txt` | +| **Ур.4: Конкуренция** | `load_parallel.sh` | 12/12 загрузок приняты и обработаны за 2.11 с, RAM worker 36 MiB / 512 MiB | `evidence/F_parallel.txt` | +| **Ур.4: Очистка staging** | `fault_staging_cleanup.sh` | Janitor успешно удаляет `staging/` ключи после фиксации результата (`leftover = 0`) | `evidence/G_staging.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Условия:** MacBook Air (Apple Silicon), 1 хост. Лимиты: API 512MB, Worker 512MB, PG 256MB. +**Нагрузка:** 12 параллельных загрузок одного файла (closed-loop). +**Результат:** 12/12 успешно обработаны за 2.11 с. Пиковое потребление RAM worker: 36.43 MiB (7.1%). Ошибок нет. +**Ограничения измерения:** Один прогон, синтетический фикстура, отсутствие прогрева. Не измерялись p95/p99 RPS (относится к треку 5). -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Воздействие | Ожидаем | Наблюдали | Потеря данных | +|---|---|---|---| +| Смерть worker с lease | Возврат задачи после таймаута | `requeued` через 5с, успешный retry (attempt=2) | 0 | +| Crash после COMMIT, до ack | Повторная доставка = no-op | `reclaimed` → `status=done, ack`, дублей вариантов 0 | 0 | +| Разрыв DB/Broker | Восстановление из БД | Задание `pending` в БД, автоматически опубликовано и выполнено | 0 | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Выдерживаются:** Падение 1+ реплик API, падение worker (задачи возвращаются по lease или из БД), временная недоступность S3/Redis (retry). +- **Не выдерживаются (учебные упрощения):** Потеря хоста (один Docker-хост), потеря volume PostgreSQL или RustFS (нет кросс-датацентровой репликации). +- **Не реализовано:** Строгая IAM-политика с раздельными правами на S3 для API и worker (используются единые учебные ключи). +- **Подтверждено:** Outbox-эквивалент надёжно защищает от потери намерения обработки; lease и PEL-reclaim корректно обрабатывают crash worker'а без дублирования эффектов. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d9d950f2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,198 @@ +services: + lb: + image: haproxy:2.9-alpine + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + restart: unless-stopped + + api: + build: . + expose: + - "8080" + environment: + PORT: "8080" + DATABASE_URL: postgres://${PG_USER}:${PG_PASSWORD}@postgres:5432/${PG_DB}?sslmode=disable&timezone=UTC + REDIS_URL: redis://redis:6379/0 + BROKER_URL: redis://broker:6379/1 + SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}" + IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}" + S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}" + S3_REGION: "${S3_REGION:-us-east-1}" + S3_BUCKET: "${S3_BUCKET:-messenger-attachments}" + S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}" + S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}" + S3_USE_SSL: "${S3_USE_SSL:-false}" + MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}" + MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}" + ATTACHMENT_GRACE_SECONDS: "${ATTACHMENT_GRACE_SECONDS:-600}" + JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + broker: + condition: service_healthy + s3: + condition: service_started + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health/ready"] + interval: 5s + timeout: 3s + start_period: 15s + retries: 5 + deploy: + replicas: 2 + resources: + limits: + cpus: "0.50" + memory: 512M + restart: unless-stopped + + worker: + build: . + command: ["./worker"] + environment: + DATABASE_URL: postgres://${PG_USER}:${PG_PASSWORD}@postgres:5432/${PG_DB}?sslmode=disable&timezone=UTC + REDIS_URL: redis://broker:6379/1 + S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}" + S3_REGION: "${S3_REGION:-us-east-1}" + S3_BUCKET: "${S3_BUCKET:-messenger-attachments}" + S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}" + S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}" + S3_USE_SSL: "${S3_USE_SSL:-false}" + MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}" + MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}" + JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}" + JOB_CONCURRENCY: "${JOB_CONCURRENCY:-2}" + JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-90}" + JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-60}" + JOB_POLL_SECONDS: "${JOB_POLL_SECONDS:-5}" + STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-300}" + JOB_RECLAIM_IDLE_SECONDS: "${JOB_RECLAIM_IDLE_SECONDS:-15}" + FAULT_EXIT_AT: "${FAULT_EXIT_AT:-}" + FAULT_FAIL_MODE: "${FAULT_FAIL_MODE:-}" + FAULT_FAIL_TIMES: "${FAULT_FAIL_TIMES:-1000000}" + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + s3: + condition: service_started + healthcheck: + disable: true + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 512M + restart: unless-stopped + + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: ${PG_USER} + POSTGRES_PASSWORD: ${PG_PASSWORD} + POSTGRES_DB: ${PG_DB} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"] + interval: 5s + timeout: 5s + retries: 5 + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + redis: + image: redis:7-alpine + command: > + redis-server + --appendonly yes + --appendfsync everysec + --auto-aof-rewrite-percentage 100 + --auto-aof-rewrite-min-size 64mb + --maxmemory 128mb + --maxmemory-policy noeviction + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + deploy: + resources: + limits: + cpus: "0.25" + memory: 192M + restart: unless-stopped + + broker: + image: redis:7-alpine + command: > + redis-server + --appendonly yes + --appendfsync everysec + --maxmemory 256mb + --maxmemory-policy noeviction + volumes: + - broker_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + s3: + image: rustfs/rustfs@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + command: ["server", "--address", ":9000"] + environment: + RUSTFS_ADDRESS: ":9000" + RUSTFS_VOLUMES: "/data" + RUSTFS_REGION: "${S3_REGION:-us-east-1}" + RUSTFS_ROOT_USER: "${S3_ROOT_USER:-minioadmin}" + RUSTFS_ROOT_PASSWORD: "${S3_ROOT_PASSWORD:-minioadmin-change-me-123}" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_OBS_LOGGER_LEVEL: "warn" + volumes: + - s3_data:/data + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + s3_data: + broker_data: \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..4eed24ec --- /dev/null +++ b/go.mod @@ -0,0 +1,34 @@ +module mtusi2026lab5_fork + +go 1.22.2 + +require ( + github.com/HugoSmits86/nativewebp v1.3.0 + github.com/disintegration/imaging v1.6.2 + github.com/go-chi/chi/v5 v5.0.12 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.5.5 + github.com/minio/minio-go/v7 v7.0.70 + github.com/redis/go-redis/v9 v9.5.1 + golang.org/x/crypto v0.21.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rs/xid v1.5.0 // indirect + golang.org/x/image v0.24.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.22.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..706b3649 --- /dev/null +++ b/go.sum @@ -0,0 +1,72 @@ +github.com/HugoSmits86/nativewebp v1.3.0 h1:n1egtEzSV4KwFtealr7dzdYq1wI/uj/bOQ/QcTcIyVE= +github.com/HugoSmits86/nativewebp v1.3.0/go.mod h1:YNQuWenlVmSUUASVNhTDwf4d7FwYQGbGhklC8p72Vr8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= +github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= +golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..fe761c24 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + log stdout format raw local0 info + maxconn 4096 + +defaults + mode http + timeout connect 5s + timeout client 30s + timeout server 30s + timeout check 5s + option httplog + option dontlognull + +resolvers docker + nameserver dns1 127.0.0.11:53 + hold valid 5s + hold nx 5s + hold refused 5s + hold timeout 5s + +frontend fe_main + bind *:8080 + default_backend be_api + +backend be_api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 2s fall 3 rise 2 init-addr libc,none resolvers docker + server-template api 2 api:8080 check diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go new file mode 100644 index 00000000..6a179cde --- /dev/null +++ b/src/cmd/server/main.go @@ -0,0 +1,227 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "mtusi2026lab5_fork/src/internal/handler" + "mtusi2026lab5_fork/src/internal/session" + "mtusi2026lab5_fork/src/internal/storage" + "mtusi2026lab5_fork/src/internal/store" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/google/uuid" + "github.com/redis/go-redis/v9" +) + +var instanceID = func() string { + if v := os.Getenv("INSTANCE_ID"); v != "" { + return v + } + if h, err := os.Hostname(); err == nil && h != "" { + return h + } + return "unknown" +}() + +// streamPublisher — publisher в Redis Streams. +// XAdd возвращает ответ после приёма брокером (publisher confirm), +// что отличается от consumer ack в worker. +type streamPublisher struct { + rdb *redis.Client + stream string +} + +func (p *streamPublisher) PublishJob(ctx context.Context, jobID string) error { + return p.rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: p.stream, + Values: map[string]interface{}{"job_id": jobID}, + }).Err() +} + +func RequestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rid := uuid.New().String() + w.Header().Set("X-Request-Id", rid) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), handler.RequestIDKey, rid))) + }) +} + +func InstanceIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", instanceID) + next.ServeHTTP(w, r) + }) +} + +func main() { + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + dbURL = "postgres://lab4:lab4@postgres:5432/lab4?sslmode=disable&timezone=UTC" + } + log.Printf("INFO: [%s] connecting to PostgreSQL...", instanceID) + pgStore, err := store.NewPostgres(dbURL) + if err != nil { + log.Fatalf("ERROR: [%s] postgres: %v", instanceID, err) + } + defer pgStore.Close() + log.Printf("INFO: [%s] PostgreSQL connected, migrations applied", instanceID) + + redisURL := os.Getenv("REDIS_URL") + if redisURL == "" { + redisURL = "redis://redis:6379/0" + } + sessionTTL := 24 * time.Hour + if s := os.Getenv("SESSION_TTL_SECONDS"); s != "" { + if secs, err := strconv.Atoi(s); err == nil && secs > 0 { + sessionTTL = time.Duration(secs) * time.Second + } + } + redisStore, err := session.NewRedisStore(redisURL, sessionTTL) + if err != nil { + log.Fatalf("ERROR: [%s] session redis: %v", instanceID, err) + } + defer redisStore.Close() + log.Printf("INFO: [%s] session Redis connected (TTL=%v)", instanceID, sessionTTL) + + brokerURL := os.Getenv("BROKER_URL") + if brokerURL == "" { + brokerURL = "redis://broker:6379/1" + } + bopt, err := redis.ParseURL(brokerURL) + if err != nil { + log.Fatalf("ERROR: [%s] broker url: %v", instanceID, err) + } + brokerRDB := redis.NewClient(bopt) + defer brokerRDB.Close() + bctx, bcancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := brokerRDB.Ping(bctx).Err(); err != nil { + bcancel() + log.Fatalf("ERROR: [%s] broker ping: %v", instanceID, err) + } + bcancel() + log.Printf("INFO: [%s] broker connected", instanceID) + + s3Cfg := storage.ConfigFromEnv() + s3Client, err := storage.NewClient(s3Cfg) + if err != nil { + log.Fatalf("ERROR: [%s] s3: %v", instanceID, err) + } + log.Printf("INFO: [%s] S3 connected endpoint=%s bucket=%s", instanceID, s3Cfg.Endpoint, s3Cfg.Bucket) + + publisher := &streamPublisher{rdb: brokerRDB, stream: "attachment_jobs"} + h := handler.New(pgStore, redisStore, s3Client, publisher, sessionTTL) + + recCtx, recCancel := context.WithCancel(context.Background()) + defer recCancel() + go runAttachmentReconciler(recCtx, pgStore, s3Client, s3Cfg.Grace) + + r := chi.NewRouter() + r.Use(InstanceIDMiddleware) + r.Use(RequestIDMiddleware) + r.Use(middleware.RealIP) + r.Use(middleware.Recoverer) + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + rid, _ := r.Context().Value(handler.RequestIDKey).(string) + log.Printf("INFO: [%s] %s %s %v [request_id=%s]", instanceID, r.Method, r.URL.Path, time.Since(start), rid) + }) + }) + + r.Get("/health/live", h.HealthLive) + r.Get("/health/ready", h.HealthReady) + r.Route("/api/v1", func(r chi.Router) { + r.Post("/users", h.CreateUser) + r.Post("/auth/sessions", h.CreateSession) + r.Group(func(r chi.Router) { + r.Use(h.BearerAuthMiddleware) + r.Delete("/auth/sessions/current", h.DeleteSession) + r.Get("/users/me", h.GetMe) + r.Get("/chats", h.ListChats) + r.Post("/chats", h.CreateChat) + r.Get("/chats/{chat_id}", h.GetChat) + r.Post("/chats/{chat_id}/messages", h.SendMessage) + r.Get("/chats/{chat_id}/messages", h.GetMessages) + r.Post("/chats/{chat_id}/attachments", h.UploadAttachment) + r.Get("/attachments/{attachment_id}", h.GetAttachment) + r.Get("/attachments/{attachment_id}/content", h.DownloadAttachment) + }) + }) + r.NotFound(func(w http.ResponseWriter, r *http.Request) { + rid, _ := r.Context().Value(handler.RequestIDKey).(string) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"not_found","message":"Not Found","request_id":"` + rid + `"}}`)) + }) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + srv := &http.Server{ + Addr: "0.0.0.0:" + port, + Handler: r, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + go func() { + log.Printf("INFO: [%s] server starting on port %s", instanceID, port) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("ERROR: [%s] server: %v", instanceID, err) + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + log.Printf("INFO: [%s] shutting down...", instanceID) + recCancel() + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + srv.SetKeepAlivesEnabled(false) + if err := srv.Shutdown(ctx); err != nil { + log.Fatalf("ERROR: [%s] forced shutdown: %v", instanceID, err) + } + log.Printf("INFO: [%s] exited properly", instanceID) +} + +func runAttachmentReconciler(ctx context.Context, st store.Store, cl *storage.Client, grace time.Duration) { + t := time.NewTicker(30 * time.Second) + defer t.Stop() + log.Printf("INFO: [%s] attachment reconciler started, grace=%v", instanceID, grace) + for { + select { + case <-ctx.Done(): + log.Printf("INFO: [%s] attachment reconciler stopped", instanceID) + return + case <-t.C: + items, err := st.ClaimStaleQueued(ctx, time.Now().Add(-grace)) + if err != nil { + log.Printf("ERROR: [%s] reconciler claim: %v", instanceID, err) + continue + } + for _, a := range items { + if err := cl.DeleteObject(ctx, a.ObjectKey); err != nil { + log.Printf("ERROR: [%s] reconciler delete %s: %v", instanceID, a.ObjectKey, err) + continue + } + if err := st.DeleteFailedStaleAttachment(ctx, a.ID); err != nil { + log.Printf("ERROR: [%s] reconciler row %s: %v", instanceID, a.ID, err) + continue + } + log.Printf("INFO: [%s] reconciler cleaned stale attachment %s", instanceID, a.ID) + } + } + } +} diff --git a/src/cmd/worker/main.go b/src/cmd/worker/main.go new file mode 100644 index 00000000..f18414d4 --- /dev/null +++ b/src/cmd/worker/main.go @@ -0,0 +1,586 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "image" + "image/color" + "image/draw" + "image/jpeg" + "io" + "log" + "math/rand" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "mtusi2026lab5_fork/src/internal/model" + "mtusi2026lab5_fork/src/internal/storage" + "mtusi2026lab5_fork/src/internal/store" + + "github.com/HugoSmits86/nativewebp" + "github.com/disintegration/imaging" + "github.com/redis/go-redis/v9" +) + +var instanceID = func() string { + if h, err := os.Hostname(); err == nil && h != "" { + return h + } + return "worker-unknown" +}() + +type cfg struct { + stream, group, consumer, dlq string + concurrency int + lease, jobTimeout time.Duration + pollEvery, stagingRetention time.Duration + reclaimIdle time.Duration + maxBytes, maxPixels int64 +} + +func envInt(k string, d int) int { + if v := os.Getenv(k); v != "" { + if n, e := strconv.Atoi(v); e == nil && n > 0 { + return n + } + } + return d +} + +func loadCfg() cfg { + return cfg{ + stream: "attachment_jobs", + group: "worker_group", + consumer: instanceID, + dlq: "attachment_jobs_dlq", + concurrency: envInt("JOB_CONCURRENCY", 2), + lease: time.Duration(envInt("JOB_LEASE_SECONDS", 90)) * time.Second, + jobTimeout: time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second, + pollEvery: time.Duration(envInt("JOB_POLL_SECONDS", 5)) * time.Second, + stagingRetention: time.Duration(envInt("STAGING_RETENTION_SECONDS", 300)) * time.Second, + reclaimIdle: time.Duration(envInt("JOB_RECLAIM_IDLE_SECONDS", 15)) * time.Second, + maxBytes: int64(envInt("MAX_UPLOAD_BYTES", 10485760)), + maxPixels: int64(envInt("MAX_IMAGE_PIXELS", 40000000)), + } +} + +type app struct { + pg store.Store + s3 *storage.Client + rdb *redis.Client + c cfg +} + +// ---------- управляемый fault injection; по умолчанию полностью выключен ---------- + +func faultMarker(name string) string { return "/tmp/.lab5_fault_" + name } + +// faultExit обрывает процесс в заданной точке ровно один раз на контейнер. +// Marker в /tmp переживает restart того же контейнера, поэтому после перезапуска +// работа доходит до конца без повторного обрыва. Пересоздание контейнера +// (--force-recreate) сбрасывает marker и снова «взводит» инъекцию. +func faultExit(point string) { + if os.Getenv("FAULT_EXIT_AT") != point { + return + } + m := faultMarker("exit_" + point) + if _, err := os.Stat(m); err == nil { + return + } + _ = os.WriteFile(m, []byte(time.Now().Format(time.RFC3339Nano)), 0o600) + log.Printf("FAULT: [%s] deliberate exit at %s", instanceID, point) + os.Exit(70) +} + +// faultFailure подменяет исход обработки, чтобы проверить классификацию ошибок, +// backoff с jitter, исчерпание попыток, DLQ и операторский replay. +// Счётчик лежит в файле и переживает перезапуск того же контейнера. +func faultFailure() *procErr { + mode := os.Getenv("FAULT_FAIL_MODE") + if mode != "transient" && mode != "permanent" { + return nil + } + limit := envInt("FAULT_FAIL_TIMES", 1000000) + m := faultMarker("fail_" + mode) + n := 0 + if b, err := os.ReadFile(m); err == nil { + n, _ = strconv.Atoi(strings.TrimSpace(string(b))) + } + if n >= limit { + return nil + } + _ = os.WriteFile(m, []byte(strconv.Itoa(n+1)), 0o600) + if mode == "permanent" { + return pe("invalid_image_format", fmt.Errorf("injected permanent failure #%d", n+1)) + } + return te("s3_download_failed", fmt.Errorf("injected transient failure #%d", n+1)) +} + +func main() { + log.Printf("INFO: [%s] worker starting...", instanceID) + c := loadCfg() + + dbURL := os.Getenv("DATABASE_URL") + if dbURL == "" { + dbURL = "postgres://lab4:lab4@postgres:5432/lab4?sslmode=disable&timezone=UTC" + } + pg, err := store.NewPostgres(dbURL) + if err != nil { + log.Fatalf("ERROR: [%s] postgres: %v", instanceID, err) + } + defer pg.Close() + + brokerURL := os.Getenv("REDIS_URL") + if brokerURL == "" { + brokerURL = "redis://broker:6379/1" + } + opt, err := redis.ParseURL(brokerURL) + if err != nil { + log.Fatalf("ERROR: [%s] broker url: %v", instanceID, err) + } + rdb := redis.NewClient(opt) + defer rdb.Close() + + s3c, err := storage.NewClient(storage.ConfigFromEnv()) + if err != nil { + log.Fatalf("ERROR: [%s] s3: %v", instanceID, err) + } + + a := &app{pg: pg, s3: s3c, rdb: rdb, c: c} + if err := rdb.XGroupCreateMkStream(context.Background(), c.stream, c.group, "0").Err(); err != nil && + !strings.Contains(err.Error(), "BUSYGROUP") { + log.Fatalf("ERROR: [%s] create group: %v", instanceID, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + q := make(chan os.Signal, 1) + signal.Notify(q, syscall.SIGINT, syscall.SIGTERM) + <-q + log.Printf("INFO: [%s] shutting down...", instanceID) + cancel() + }() + + log.Printf("INFO: [%s] concurrency=%d lease=%v timeout=%v reclaimIdle=%v maxBytes=%d maxPixels=%d", + instanceID, c.concurrency, c.lease, c.jobTimeout, c.reclaimIdle, c.maxBytes, c.maxPixels) + + var wg sync.WaitGroup + wg.Add(4) + go func() { defer wg.Done(); a.consume(ctx) }() + go func() { defer wg.Done(); a.recover(ctx) }() + go func() { defer wg.Done(); a.janitor(ctx) }() + go func() { defer wg.Done(); a.reclaim(ctx) }() + + <-ctx.Done() + wg.Wait() + log.Printf("INFO: [%s] worker stopped gracefully", instanceID) +} + +func (a *app) consume(ctx context.Context) { + sem := make(chan struct{}, a.c.concurrency) + for { + if ctx.Err() != nil { + return + } + res, err := a.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: a.c.group, + Consumer: a.c.consumer, + Streams: []string{a.c.stream, ">"}, + Count: int64(a.c.concurrency), + Block: 3 * time.Second, + }).Result() + if err != nil && !errors.Is(err, redis.Nil) { + if ctx.Err() != nil { + return + } + log.Printf("ERROR: [%s] XReadGroup: %v", instanceID, err) + time.Sleep(time.Second) + continue + } + for _, st := range res { + for _, m := range st.Messages { + jobID, _ := m.Values["job_id"].(string) + if jobID == "" { + a.ack(ctx, m.ID) + continue + } + sem <- struct{}{} + go func(msgID, jid string) { + defer func() { <-sem }() + a.handle(ctx, msgID, jid) + }(m.ID, jobID) + } + } + } +} + +// recover возвращает зависшие lease и переопубликует намерения из БД. +// Это эквивалент outbox: задание и метаданные записаны одной транзакцией, +// поэтому разрыв COMMIT/publish восстанавливается без ручного создания job. +func (a *app) recover(ctx context.Context) { + t := time.NewTicker(a.c.pollEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if n, err := a.pg.RequeueExpiredLeases(ctx); err != nil { + log.Printf("ERROR: [%s] requeue leases: %v", instanceID, err) + } else if n > 0 { + log.Printf("WARN: [%s] requeued %d expired-lease jobs", instanceID, n) + } + + jobs, err := a.pg.GetPendingJobs(ctx, 20) + if err != nil { + log.Printf("ERROR: [%s] pending jobs: %v", instanceID, err) + continue + } + for _, j := range jobs { + if err := a.publish(ctx, j.ID); err != nil { + log.Printf("WARN: [%s] republish %s: %v", instanceID, j.ID, err) + continue + } + _ = a.pg.UpdateJobStatus(ctx, j.ID, "published", j.Attempts, "") + log.Printf("INFO: [%s] republished job %s (attempt=%d)", instanceID, j.ID, j.Attempts) + } + } + } +} + +// reclaim подхватывает сообщения, оставшиеся неподтверждёнными после crash. +// Это повторная доставка ТОГО ЖЕ job_id, а не создание нового задания. +// Преждевременный ack всё ещё обрабатывающейся работы безопасен: эффект +// защищён lease в БД, а recover вернёт job, если он не терминальный. +func (a *app) reclaim(ctx context.Context) { + t := time.NewTicker(a.c.pollEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + pend, err := a.rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: a.c.stream, + Group: a.c.group, + Start: "-", + End: "+", + Count: 20, + Idle: a.c.reclaimIdle, + }).Result() + if err != nil && !errors.Is(err, redis.Nil) { + log.Printf("ERROR: [%s] XPendingExt: %v", instanceID, err) + continue + } + for _, p := range pend { + msgs, err := a.rdb.XClaim(ctx, &redis.XClaimArgs{ + Stream: a.c.stream, + Group: a.c.group, + Consumer: a.c.consumer, + MinIdle: a.c.reclaimIdle, + Messages: []string{p.ID}, + }).Result() + if err != nil { + log.Printf("WARN: [%s] XClaim %s: %v", instanceID, p.ID, err) + continue + } + for _, m := range msgs { + jobID, _ := m.Values["job_id"].(string) + if jobID == "" { + a.ack(ctx, m.ID) + continue + } + log.Printf("INFO: [%s] reclaimed message %s job=%s idle=%v", + instanceID, m.ID, jobID, p.Idle) + a.handle(ctx, m.ID, jobID) + } + } + } + } +} + +// janitor дочищает staging, оставшийся после crash между COMMIT и удалением. +func (a *app) janitor(ctx context.Context) { + t := time.NewTicker(60 * time.Second) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + refs, err := a.pg.ListCompletedStaging(ctx, time.Now().Add(-a.c.stagingRetention)) + if err != nil { + log.Printf("ERROR: [%s] list staging: %v", instanceID, err) + continue + } + for _, r := range refs { + if err := a.s3.DeleteObject(ctx, r.StagingKey); err != nil { + continue + } + _ = a.pg.ClearStagingKey(ctx, r.ID) + log.Printf("INFO: [%s] janitor removed staging %s", instanceID, r.StagingKey) + } + } + } +} + +func (a *app) publish(ctx context.Context, jobID string) error { + return a.rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: a.c.stream, + Values: map[string]interface{}{"job_id": jobID}, + }).Err() +} + +func (a *app) ack(ctx context.Context, msgID string) { + _ = a.rdb.XAck(ctx, a.c.stream, a.c.group, msgID).Err() +} + +func (a *app) handle(ctx context.Context, msgID, jobID string) { + job, err := a.pg.ClaimJob(ctx, jobID, a.c.consumer, a.c.lease) + if err != nil { + if errors.Is(err, store.ErrConflict) { + if cur, e := a.pg.GetJobByID(ctx, jobID); e == nil { + log.Printf("INFO: [%s] job %s not claimable (status=%s), ack", instanceID, jobID, cur.Status) + } + a.ack(ctx, msgID) + return + } + log.Printf("ERROR: [%s] claim %s: %v", instanceID, jobID, err) + return + } + + faultExit("after_claim") // lease взят, работа ещё не сделана + + att, err := a.pg.GetAttachmentByID(ctx, job.AttachmentID) + if err != nil { + a.fail(ctx, msgID, job, job.AttachmentID, "attachment_not_found", err, false) + return + } + if att.Status == "ready" { + log.Printf("INFO: [%s] job %s already complete, no-op ack", instanceID, jobID) + _ = a.pg.CompleteAttachment(ctx, att.ID, jobID, att.ObjectKey, nil) + a.ack(ctx, msgID) + return + } + if job.Attempts > job.MaxAttempts { + a.fail(ctx, msgID, job, att.ID, "retry_exhausted", + fmt.Errorf("attempts=%d max=%d", job.Attempts, job.MaxAttempts), false) + return + } + + jctx, cancel := context.WithTimeout(ctx, a.c.jobTimeout) + defer cancel() + + variants, finalKey, perr := a.build(jctx, att) + if perr != nil { + a.fail(jctx, msgID, job, att.ID, perr.code, perr.err, !perr.permanent) + return + } + + faultExit("after_s3_before_commit") // объекты в S3 есть, результата в БД нет + + if err := a.pg.CompleteAttachment(ctx, att.ID, jobID, finalKey, variants); err != nil { + a.fail(ctx, msgID, job, att.ID, "db_commit_failed", err, true) + return + } + + faultExit("after_commit_before_ack") // результат зафиксирован, ack не отправлен + + // staging удаляем только после устойчивой фиксации результата + if att.ObjectKey != finalKey { + if err := a.s3.DeleteObject(ctx, att.ObjectKey); err == nil { + _ = a.pg.ClearStagingKey(ctx, att.ID) + } + } + a.ack(ctx, msgID) + log.Printf("INFO: [%s] job %s done (attempt=%d, variants=%d)", instanceID, jobID, job.Attempts, len(variants)) +} + +type procErr struct { + code string + err error + permanent bool +} + +func pe(code string, err error) *procErr { return &procErr{code, err, true} } +func te(code string, err error) *procErr { return &procErr{code, err, false} } + +func (a *app) build(ctx context.Context, att *model.Attachment) ([]model.Variant, string, *procErr) { + if fe := faultFailure(); fe != nil { + return nil, "", fe + } + + obj, err := a.s3.GetObject(ctx, att.ObjectKey) + if err != nil { + return nil, "", te("s3_download_failed", err) + } + raw, err := io.ReadAll(io.LimitReader(obj.Body, a.c.maxBytes+1)) + obj.Body.Close() + if err != nil { + return nil, "", te("s3_read_failed", err) + } + if int64(len(raw)) > a.c.maxBytes { + return nil, "", pe("image_too_large", fmt.Errorf("bytes=%d", len(raw))) + } + + dcfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err != nil { + return nil, "", pe("invalid_image_format", err) + } + if int64(dcfg.Width)*int64(dcfg.Height) > a.c.maxPixels { + return nil, "", pe("image_too_many_pixels", fmt.Errorf("%dx%d", dcfg.Width, dcfg.Height)) + } + + img, err := imaging.Decode(bytes.NewReader(raw), imaging.AutoOrientation(true)) + if err != nil { + return nil, "", pe("invalid_image_format", err) + } + + ext := ".png" + if att.ContentType == "image/jpeg" { + ext = ".jpg" + } + finalKey := "original/" + att.ID + ext + thumbKey := "variants/" + att.ID + "/thumbnail.jpg" + cropKey := "variants/" + att.ID + "/crop.jpg" + optKey := "variants/" + att.ID + "/optimized.webp" + + // оригинал публикуется байт-в-байт, без перекодирования + if err := a.s3.PutObject(ctx, finalKey, raw, att.ContentType); err != nil { + return nil, "", te("s3_upload_original_failed", err) + } + + thumb := imaging.Fit(img, 256, 256, imaging.Lanczos) + tb, err := encodeJPEG(thumb, 85) + if err != nil { + return nil, "", pe("jpeg_encode_failed", err) + } + crop := imaging.Fill(img, 128, 128, imaging.Center, imaging.Lanczos) + cb, err := encodeJPEG(crop, 90) + if err != nil { + return nil, "", pe("jpeg_encode_failed", err) + } + src := img + if img.Bounds().Dx() > 1280 || img.Bounds().Dy() > 1280 { + src = imaging.Fit(img, 1280, 1280, imaging.Lanczos) + } + ob, err := encodeWebP(src) + if err != nil { + return nil, "", pe("webp_encode_failed", err) + } + + for _, o := range []struct { + key, ct string + data []byte + }{ + {thumbKey, "image/jpeg", tb}, + {cropKey, "image/jpeg", cb}, + {optKey, "image/webp", ob}, + } { + if err := a.s3.PutObject(ctx, o.key, o.data, o.ct); err != nil { + return nil, "", te("s3_upload_variant_failed", err) + } + } + + mk := func(name, key, ct string, b []byte, im image.Image) model.Variant { + return model.Variant{ + Name: name, ObjectKey: key, ContentType: ct, + SizeBytes: int64(len(b)), Width: im.Bounds().Dx(), Height: im.Bounds().Dy(), + SHA256: sha256Hex(b), + } + } + return []model.Variant{ + mk("thumbnail", thumbKey, "image/jpeg", tb, thumb), + mk("crop", cropKey, "image/jpeg", cb, crop), + mk("optimized", optKey, "image/webp", ob, src), + }, finalKey, nil +} + +func flatten(src image.Image) *image.RGBA { + b := src.Bounds() + dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(dst, dst.Bounds(), image.NewUniform(color.RGBA{255, 255, 255, 255}), image.Point{}, draw.Src) + draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Over) + return dst +} + +func encodeJPEG(src image.Image, q int) ([]byte, error) { + var buf bytes.Buffer + if err := jpeg.Encode(&buf, flatten(src), &jpeg.Options{Quality: q}); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func encodeWebP(src image.Image) ([]byte, error) { + var buf bytes.Buffer + if err := nativewebp.Encode(&buf, src, nil); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (a *app) fail(ctx context.Context, msgID string, job *model.Job, attID, code string, cause error, transient bool) { + log.Printf("ERROR: [%s] job %s %s: %v (attempt=%d/%d)", + instanceID, job.ID, code, cause, job.Attempts, job.MaxAttempts) + + if transient && job.Attempts < job.MaxAttempts { + delay := backoff(job.Attempts) + if err := a.pg.RetryJob(ctx, job.ID, attID, code, cause.Error(), delay); err != nil { + log.Printf("ERROR: [%s] retry job: %v", instanceID, err) + return + } + log.Printf("WARN: [%s] job %s retry in %v", instanceID, job.ID, delay) + a.ack(ctx, msgID) + return + } + + if err := a.pg.FailJob(ctx, job.ID, attID, code, cause.Error()); err != nil { + log.Printf("ERROR: [%s] fail job: %v", instanceID, err) + return + } + // DLQ: реестр для оператора; bytes и credentials не передаём + _ = a.rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: a.c.dlq, + Values: map[string]interface{}{ + "job_id": job.ID, "attachment_id": attID, + "error_code": code, "attempts": job.Attempts, + }, + }).Err() + a.ack(ctx, msgID) +} + +// backoff: 5s * 2^(n-1), cap 60s, jitter ±25% +func backoff(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + d := float64(5*time.Second) * float64(int(1)< float64(60*time.Second) { + d = float64(60 * time.Second) + } + j := 0.75 + rand.Float64()*0.5 + return time.Duration(d * j) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func sha256Hex(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} diff --git a/src/internal/auth/auth.go b/src/internal/auth/auth.go new file mode 100644 index 00000000..57942bb3 --- /dev/null +++ b/src/internal/auth/auth.go @@ -0,0 +1,46 @@ +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "time" + + "golang.org/x/crypto/bcrypt" +) + +func HashPassword(password string) (string, error) { + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + return string(hash), nil +} + +func CheckPassword(password, hash string) bool { + err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) + return err == nil +} + +func GenerateToken() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(b), nil +} + +func HashToken(token string) string { + h := sha256.Sum256([]byte(token)) + return hex.EncodeToString(h[:]) +} + +func SecureCompare(a, b string) bool { + return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 +} + +func IsSessionExpired(expiresAt time.Time) bool { + return time.Now().After(expiresAt) +} diff --git a/src/internal/handler/handler.go b/src/internal/handler/handler.go new file mode 100644 index 00000000..c5a2bdb1 --- /dev/null +++ b/src/internal/handler/handler.go @@ -0,0 +1,888 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "mtusi2026lab5_fork/src/internal/auth" + "mtusi2026lab5_fork/src/internal/model" + "mtusi2026lab5_fork/src/internal/storage" + "mtusi2026lab5_fork/src/internal/store" + + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +type CtxKey string + +const RequestIDKey CtxKey = "request_id" +const UserIDKey CtxKey = "user_id" + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) +var pngSignature = []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A} + +var ( + errUnsupportedImage = errors.New("unsupported_image") + errCorruptImage = errors.New("corrupt_image") +) + +type SessionStore interface { + Get(ctx context.Context, tokenHash string) (string, error) + Set(ctx context.Context, userID, tokenHash string, ttl time.Duration) error + Delete(ctx context.Context, tokenHash string) error +} + +// JobPublisher публикует задание в поток брокера. +// Реализация на базе Redis находится в main.go или в отдельном пакете. +type JobPublisher interface { + PublishJob(ctx context.Context, jobID string) error +} + +type Handler struct { + store store.Store + sessionStore SessionStore + s3 *storage.Client + publisher JobPublisher + sessionTTL time.Duration +} + +func New(s store.Store, ss SessionStore, s3 *storage.Client, pub JobPublisher, ttl time.Duration) *Handler { + return &Handler{ + store: s, + sessionStore: ss, + s3: s3, + publisher: pub, + sessionTTL: ttl, + } +} + +func isValidUUID(s string) bool { + _, err := uuid.Parse(s) + return err == nil +} + +func userToMap(u *model.User) map[string]interface{} { + return map[string]interface{}{ + "id": u.ID, + "username": u.Username, + "display_name": u.DisplayName, + "created_at": u.CreatedAt.UTC().Format(time.RFC3339Nano), + } +} + +func chatToMap(c *model.Chat) map[string]interface{} { + ids := c.MemberIDs + if ids == nil { + ids = []string{} + } + return map[string]interface{}{ + "id": c.ID, + "title": c.Title, + "member_ids": ids, + "created_at": c.CreatedAt.UTC().Format(time.RFC3339Nano), + } +} + +func messageToMap(m *model.Message) map[string]interface{} { + ids := m.AttachmentIDs + if ids == nil { + ids = []string{} + } + return map[string]interface{}{ + "id": m.ID, + "chat_id": m.ChatID, + "sender_id": m.SenderID, + "text": m.Text, + "attachment_ids": ids, + "created_at": m.CreatedAt.UTC().Format(time.RFC3339Nano), + } +} + +func attachmentToMap(a *model.Attachment) map[string]interface{} { + variants := []any{} + if a.Variants != nil { + variants = a.Variants + } + var errorCode interface{} = nil + if a.ErrorCode != nil { + errorCode = *a.ErrorCode + } + return map[string]interface{}{ + "id": a.ID, + "chat_id": a.ChatID, + "owner_id": a.OwnerID, + "filename": a.Filename, + "content_type": a.ContentType, + "size_bytes": a.SizeBytes, + "sha256": a.SHA256, + "status": a.Status, + "variants": variants, + "error_code": errorCode, + "created_at": a.CreatedAt.UTC().Format(time.RFC3339Nano), + "updated_at": a.UpdatedAt.UTC().Format(time.RFC3339Nano), + } +} + +func respondError(w http.ResponseWriter, r *http.Request, code int, errorCode string) { + requestID, _ := r.Context().Value(RequestIDKey).(string) + log.Printf("ERROR: %d - %s [request_id=%s]", code, errorCode, requestID) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(map[string]interface{}{ + "error": map[string]string{ + "code": errorCode, + "message": http.StatusText(code), + "request_id": requestID, + }, + }) +} + +func respondJSON(w http.ResponseWriter, code int, data interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(data) +} + +func checkContentType(r *http.Request) bool { + ct := r.Header.Get("Content-Type") + return ct == "application/json" || strings.HasPrefix(ct, "application/json;") +} + +func validateFilename(name string) bool { + if name == "" || utf8.RuneCountInString(name) > 255 { + return false + } + for _, ch := range name { + if ch == '/' || ch == '\\' || ch == 0 || unicode.IsControl(ch) { + return false + } + } + return true +} + +func detectImage(data []byte) (string, image.Config, error) { + if len(data) == 0 { + return "", image.Config{}, errUnsupportedImage + } + if bytes.HasPrefix(data, pngSignature) { + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 { + return "", image.Config{}, errCorruptImage + } + return "image/png", cfg, nil + } + if len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 { + return "", image.Config{}, errCorruptImage + } + return "image/jpeg", cfg, nil + } + return "", image.Config{}, errUnsupportedImage +} + +func verifyImageIntegrity(data []byte) error { + _, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return errCorruptImage + } + return nil +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) { + if !checkContentType(r) { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + var req struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if !usernamePattern.MatchString(req.Username) || + utf8.RuneCountInString(req.Password) < 12 || + utf8.RuneCountInString(req.Password) > 128 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + pwHash, err := auth.HashPassword(req.Password) + if err != nil { + log.Printf("ERROR: hash password: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + user, err := h.store.CreateUser(r.Context(), req.Username, req.DisplayName, pwHash) + if err != nil { + if errors.Is(err, store.ErrConflict) { + respondError(w, r, http.StatusConflict, "conflict") + } else { + respondError(w, r, http.StatusBadRequest, "invalid_request") + } + return + } + respondJSON(w, http.StatusCreated, userToMap(user)) +} + +func (h *Handler) CreateSession(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if !strings.HasPrefix(authHeader, "Basic ") { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + payload, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Basic ")) + if err != nil { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + parts := strings.SplitN(string(payload), ":", 2) + if len(parts) != 2 { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + username, password := parts[0], parts[1] + user, err := h.store.GetUserByUsername(r.Context(), username) + if err != nil || !auth.CheckPassword(password, user.PasswordHash) { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + token, err := auth.GenerateToken() + if err != nil { + log.Printf("ERROR: generate token: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + tokenHash := auth.HashToken(token) + expiresAt := time.Now().Add(h.sessionTTL) + + if _, err := h.store.CreateSession(r.Context(), user.ID, tokenHash, expiresAt); err != nil { + log.Printf("ERROR: create postgres session: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + if err := h.sessionStore.Set(r.Context(), user.ID, tokenHash, h.sessionTTL); err != nil { + _ = h.store.DeleteSession(r.Context(), tokenHash) + log.Printf("ERROR: create redis session: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + respondJSON(w, http.StatusCreated, map[string]interface{}{ + "token": token, + "token_type": "Bearer", + "expires_at": expiresAt.UTC().Format(time.RFC3339Nano), + "user": userToMap(user), + }) +} + +func (h *Handler) DeleteSession(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if !strings.HasPrefix(authHeader, "Bearer ") { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + token := strings.TrimPrefix(authHeader, "Bearer ") + tokenHash := auth.HashToken(token) + if err := h.store.DeleteSession(r.Context(), tokenHash); err != nil { + log.Printf("ERROR: delete postgres session: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + if err := h.sessionStore.Delete(r.Context(), tokenHash); err != nil { + log.Printf("WARN: delete redis session: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) BearerAuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if !strings.HasPrefix(authHeader, "Bearer ") { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + token := strings.TrimPrefix(authHeader, "Bearer ") + tokenHash := auth.HashToken(token) + userID, err := h.sessionStore.Get(r.Context(), tokenHash) + if err != nil { + sess, errPg := h.store.GetSession(r.Context(), tokenHash) + if errPg != nil { + if errors.Is(errPg, store.ErrNotFound) { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + log.Printf("ERROR: postgres session lookup: %v [request_id=%s]", errPg, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + if auth.IsSessionExpired(sess.ExpiresAt) { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + userID = sess.UserID + } + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserIDKey, userID))) + }) +} + +func (h *Handler) GetMe(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + user, err := h.store.GetUserByID(r.Context(), userID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + log.Printf("ERROR: get user by id: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + respondJSON(w, http.StatusOK, userToMap(user)) +} + +func (h *Handler) CreateChat(w http.ResponseWriter, r *http.Request) { + if !checkContentType(r) { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + var req struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if len(req.MemberIDs) == 0 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + seen := make(map[string]struct{}, len(req.MemberIDs)) + isMember := false + for _, id := range req.MemberIDs { + if !isValidUUID(id) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if _, exists := seen[id]; exists { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + seen[id] = struct{}{} + if id == userID { + isMember = true + } + if _, err := h.store.GetUserByID(r.Context(), id); err != nil { + if errors.Is(err, store.ErrNotFound) { + respondError(w, r, http.StatusNotFound, "not_found") + return + } + log.Printf("ERROR: validate chat member: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + } + if !isMember { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + chat, err := h.store.CreateChat(r.Context(), req.Title, req.MemberIDs) + if err != nil { + switch { + case errors.Is(err, store.ErrInvalidRequest), errors.Is(err, store.ErrNotFound): + respondError(w, r, http.StatusBadRequest, "invalid_request") + case errors.Is(err, store.ErrConflict): + respondError(w, r, http.StatusConflict, "conflict") + default: + log.Printf("ERROR: create chat: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + respondJSON(w, http.StatusCreated, chatToMap(chat)) +} + +func (h *Handler) ListChats(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + limit := 50 + if ls := r.URL.Query().Get("limit"); ls != "" { + p, err := strconv.Atoi(ls) + if err != nil || p < 1 || p > 100 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + limit = p + } + chats, nextCursor, err := h.store.ListChats(r.Context(), userID, limit, r.URL.Query().Get("cursor")) + if err != nil { + switch { + case errors.Is(err, store.ErrInvalidRequest): + respondError(w, r, http.StatusBadRequest, "invalid_request") + case errors.Is(err, store.ErrNotFound): + respondError(w, r, http.StatusNotFound, "not_found") + default: + log.Printf("ERROR: list chats: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + items := make([]map[string]interface{}, 0, len(chats)) + for _, c := range chats { + items = append(items, chatToMap(c)) + } + respondJSON(w, http.StatusOK, map[string]interface{}{ + "items": items, + "next_cursor": nextCursor, + }) +} + +func (h *Handler) GetChat(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + chatID := chi.URLParam(r, "chat_id") + if !isValidUUID(chatID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + chat, err := h.store.GetChat(r.Context(), chatID, userID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + respondError(w, r, http.StatusNotFound, "not_found") + return + } + log.Printf("ERROR: get chat: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + respondJSON(w, http.StatusOK, chatToMap(chat)) +} + +func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) { + if !checkContentType(r) { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + chatID := chi.URLParam(r, "chat_id") + if !isValidUUID(chatID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + idemKey := r.Header.Get("Idempotency-Key") + if !idempotencyKeyPattern.MatchString(idemKey) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + + var req struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + + // Нормализация: nil → [] + if req.AttachmentIDs == nil { + req.AttachmentIDs = []string{} + } + + if utf8.RuneCountInString(req.Text) > 4000 || len(req.AttachmentIDs) > 10 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + + seen := make(map[string]struct{}, len(req.AttachmentIDs)) + normalized := make([]string, 0, len(req.AttachmentIDs)) + for _, rawID := range req.AttachmentIDs { + parsed, err := uuid.Parse(rawID) + if err != nil { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + id := parsed.String() + if _, exists := seen[id]; exists { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + seen[id] = struct{}{} + normalized = append(normalized, id) + } + req.AttachmentIDs = normalized + + // КРИТИЧНО: канонический хеш через СТРУКТУРУ, а не map! + // В map порядок ключей недетерминирован, что ломает идемпотентность. + canonicalBody, err := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{ + Text: req.Text, + AttachmentIDs: req.AttachmentIDs, + }) + if err != nil { + log.Printf("ERROR: marshal canonical message: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + + requestHash := sha256Hex(canonicalBody) + path := "/api/v1/chats/" + chatID + "/messages" + + msg, err := h.store.SendMessageIdempotent( + r.Context(), chatID, userID, + req.Text, req.AttachmentIDs, + idemKey, path, requestHash, + ) + if err != nil { + switch { + case errors.Is(err, store.ErrInvalidRequest): + respondError(w, r, http.StatusBadRequest, "invalid_request") + case errors.Is(err, store.ErrNotFound): + respondError(w, r, http.StatusNotFound, "not_found") + case errors.Is(err, store.ErrConflict): + respondError(w, r, http.StatusConflict, "conflict") + default: + log.Printf("ERROR: send message: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + if msg.AttachmentIDs == nil { + msg.AttachmentIDs = []string{} + } + respondJSON(w, http.StatusCreated, messageToMap(msg)) +} + +func (h *Handler) GetMessages(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + chatID := chi.URLParam(r, "chat_id") + if !isValidUUID(chatID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + limit := 50 + if ls := r.URL.Query().Get("limit"); ls != "" { + p, err := strconv.Atoi(ls) + if err != nil || p < 1 || p > 100 { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + limit = p + } + messages, nextCursor, err := h.store.GetMessages(r.Context(), chatID, userID, r.URL.Query().Get("cursor"), limit) + if err != nil { + switch { + case errors.Is(err, store.ErrInvalidRequest): + respondError(w, r, http.StatusBadRequest, "invalid_request") + case errors.Is(err, store.ErrNotFound): + respondError(w, r, http.StatusNotFound, "not_found") + default: + log.Printf("ERROR: get messages: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + items := make([]map[string]interface{}, 0, len(messages)) + for _, m := range messages { + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + items = append(items, messageToMap(m)) + } + respondJSON(w, http.StatusOK, map[string]interface{}{ + "items": items, + "next_cursor": nextCursor, + }) +} + +func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + if h.s3 == nil { + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + chatID := chi.URLParam(r, "chat_id") + if !isValidUUID(chatID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + filename := r.URL.Query().Get("filename") + if !validateFilename(filename) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if _, err := h.store.GetChat(r.Context(), chatID, userID); err != nil { + if errors.Is(err, store.ErrNotFound) { + respondError(w, r, http.StatusNotFound, "not_found") + return + } + log.Printf("ERROR: chat membership for upload: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + + cfg := h.s3.Config() + data, err := io.ReadAll(io.LimitReader(r.Body, cfg.MaxUpload+1)) + if err != nil { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + if int64(len(data)) > cfg.MaxUpload { + respondError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large") + return + } + + contentType, imgCfg, err := detectImage(data) + if err != nil { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + if ct := r.Header.Get("Content-Type"); ct != "" && !strings.HasPrefix(ct, contentType) { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + if int64(imgCfg.Width)*int64(imgCfg.Height) > cfg.MaxPixels { + respondError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large") + return + } + if err := verifyImageIntegrity(data); err != nil { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + + sha := sha256Hex(data) + + if existing, _ := h.store.GetAttachmentBySHA256InChat(r.Context(), chatID, sha); existing != nil { + if full, err := h.store.GetAttachmentForMetadata(r.Context(), existing.ID, userID); err == nil { + log.Printf("INFO: dedup hit (same chat): sha256=%s [request_id=%s]", sha[:16], r.Context().Value(RequestIDKey)) + respondJSON(w, http.StatusAccepted, attachmentToMap(full)) + return + } + } + + ext := ".png" + if contentType == "image/jpeg" { + ext = ".jpg" + } + stagingKey := "staging/" + uuid.NewString() + ext + + // Durable staging: полные проверенные байты до ответа 202. + if err := h.s3.PutObject(r.Context(), stagingKey, data, contentType); err != nil { + log.Printf("ERROR: s3 put staging: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + + att, job, err := h.store.CreateAttachmentAndJob( + r.Context(), chatID, userID, filename, + stagingKey, contentType, int64(len(data)), sha, + ) + if err != nil { + _ = h.s3.DeleteObject(r.Context(), stagingKey) + log.Printf("ERROR: create attachment and job: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + + if h.publisher != nil { + if err := h.publisher.PublishJob(r.Context(), job.ID); err != nil { + log.Printf("WARN: publish job %s failed, recover will retry: %v [request_id=%s]", + job.ID, err, r.Context().Value(RequestIDKey)) + } else { + _ = h.store.PublishJobToBroker(r.Context(), job.ID) + log.Printf("INFO: job %s published [request_id=%s]", job.ID, r.Context().Value(RequestIDKey)) + } + } + + respondJSON(w, http.StatusAccepted, attachmentToMap(att)) +} + +func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + attachmentID := chi.URLParam(r, "attachment_id") + if !isValidUUID(attachmentID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + att, err := h.store.GetAttachmentForMetadata(r.Context(), attachmentID, userID) + if err != nil { + switch { + case errors.Is(err, store.ErrNotFound): + respondError(w, r, http.StatusNotFound, "not_found") + case errors.Is(err, store.ErrInvalidRequest): + respondError(w, r, http.StatusBadRequest, "invalid_request") + default: + log.Printf("ERROR: get attachment: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + respondJSON(w, http.StatusOK, attachmentToMap(att)) +} + +func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) { + userID, ok := r.Context().Value(UserIDKey).(string) + if !ok || !isValidUUID(userID) { + respondError(w, r, http.StatusUnauthorized, "unauthorized") + return + } + if h.s3 == nil { + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + return + } + attachmentID := chi.URLParam(r, "attachment_id") + if !isValidUUID(attachmentID) { + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + + variantName := r.URL.Query().Get("variant") + switch variantName { + case "", "thumbnail", "crop", "optimized": + default: + respondError(w, r, http.StatusBadRequest, "invalid_request") + return + } + + var objectKey, contentType string + var sizeBytes int64 + + if variantName == "" { + att, err := h.store.GetAttachmentForDownload(r.Context(), attachmentID, userID) + if err != nil { + h.respondAttachmentReadError(w, r, attachmentID, userID, err) + return + } + objectKey, contentType, sizeBytes = att.ObjectKey, att.ContentType, att.SizeBytes + } else { + v, err := h.store.GetAttachmentVariant(r.Context(), attachmentID, userID, variantName) + if err != nil { + h.respondAttachmentReadError(w, r, attachmentID, userID, err) + return + } + objectKey, contentType, sizeBytes = v.ObjectKey, v.ContentType, v.SizeBytes + } + + obj, err := h.s3.GetObject(r.Context(), objectKey) + if err != nil { + switch { + case errors.Is(err, storage.ErrObjectNotFound): + respondError(w, r, http.StatusNotFound, "not_found") + case errors.Is(err, storage.ErrDependencyUnavailable): + respondError(w, r, http.StatusServiceUnavailable, "unavailable") + default: + log.Printf("ERROR: s3 get object: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } + return + } + defer obj.Body.Close() + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10)) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, obj.Body); err != nil { + log.Printf("ERROR: copy attachment body: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + } +} + +func (h *Handler) respondAttachmentReadError(w http.ResponseWriter, r *http.Request, attachmentID, userID string, err error) { + switch { + case errors.Is(err, store.ErrNotFound): + meta, metaErr := h.store.GetAttachmentForMetadata(r.Context(), attachmentID, userID) + if metaErr == nil && meta != nil && meta.Status != "ready" { + respondError(w, r, http.StatusConflict, "conflict") + return + } + respondError(w, r, http.StatusNotFound, "not_found") + case errors.Is(err, store.ErrInvalidRequest): + respondError(w, r, http.StatusBadRequest, "invalid_request") + default: + log.Printf("ERROR: read attachment: %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + } +} + +func (h *Handler) HealthLive(w http.ResponseWriter, r *http.Request) { + respondJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (h *Handler) HealthReady(w http.ResponseWriter, r *http.Request) { + respondJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} diff --git a/src/internal/model/model.go b/src/internal/model/model.go new file mode 100644 index 00000000..43bf1827 --- /dev/null +++ b/src/internal/model/model.go @@ -0,0 +1,79 @@ +package model + +import "time" + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + PasswordHash string `json:"-"` + CreatedAt time.Time `json:"created_at"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Session struct { + ID string `json:"-"` + UserID string `json:"-"` + TokenHash string `json:"-"` + ExpiresAt time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ObjectKey string `json:"-"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type Job struct { + ID string + AttachmentID string + Status string + Attempts int + MaxAttempts int + NextRunAt time.Time + LastError *string + CreatedAt time.Time + LeaseUntil *time.Time + LeasedBy *string + PipelineVersion int +} + +type StagingRef struct { + ID string + StagingKey string +} +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + ObjectKey string `json:"-"` +} diff --git a/src/internal/session/redis_store.go b/src/internal/session/redis_store.go new file mode 100644 index 00000000..852f3529 --- /dev/null +++ b/src/internal/session/redis_store.go @@ -0,0 +1,47 @@ +package session + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +type RedisStore struct { + client *redis.Client + ttl time.Duration +} + +func NewRedisStore(url string, ttl time.Duration) (*RedisStore, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + + client := redis.NewClient(opt) + + if err := client.Ping(context.Background()).Err(); err != nil { + return nil, err + } + + return &RedisStore{ + client: client, + ttl: ttl, + }, nil +} + +func (s *RedisStore) Get(ctx context.Context, tokenHash string) (string, error) { + return s.client.Get(ctx, "session:"+tokenHash).Result() +} + +func (s *RedisStore) Set(ctx context.Context, userID, tokenHash string, ttl time.Duration) error { + return s.client.Set(ctx, "session:"+tokenHash, userID, ttl).Err() +} + +func (s *RedisStore) Delete(ctx context.Context, tokenHash string) error { + return s.client.Del(ctx, "session:"+tokenHash).Err() +} + +func (s *RedisStore) Close() error { + return s.client.Close() +} diff --git a/src/internal/storage/s3.go b/src/internal/storage/s3.go new file mode 100644 index 00000000..e5e73d08 --- /dev/null +++ b/src/internal/storage/s3.go @@ -0,0 +1,230 @@ +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + ErrDependencyUnavailable = errors.New("s3_unavailable") + ErrObjectNotFound = errors.New("object_not_found") +) + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseSSL bool + MaxUpload int64 + MaxPixels int64 + Grace time.Duration +} + +type Client struct { + client *minio.Client + bucket string + cfg Config +} + +func ConfigFromEnv() Config { + endpoint := getenv("S3_ENDPOINT", "http://s3:9000") + region := getenv("S3_REGION", "us-east-1") + bucket := getenv("S3_BUCKET", "messenger-attachments") + access := getenv("S3_ACCESS_KEY_ID", "") + secret := getenv("S3_SECRET_ACCESS_KEY", "") + + useSSL := false + if v := getenv("S3_USE_SSL", "false"); v != "" { + parsed, err := strconv.ParseBool(v) + if err == nil { + useSSL = parsed + } + } + + if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { + u, err := url.Parse(endpoint) + if err == nil { + endpoint = u.Host + if u.Scheme == "https" { + useSSL = true + } + } + } + + maxUpload := getInt64("MAX_UPLOAD_BYTES", 10*1024*1024) + maxPixels := getInt64("MAX_IMAGE_PIXELS", 40_000_000) + grace := time.Duration(getInt64("ATTACHMENT_GRACE_SECONDS", 600)) * time.Second + + return Config{ + Endpoint: endpoint, + Region: region, + Bucket: bucket, + AccessKey: access, + SecretKey: secret, + UseSSL: useSSL, + MaxUpload: maxUpload, + MaxPixels: maxPixels, + Grace: grace, + } +} + +func NewClient(cfg Config) (*Client, error) { + if cfg.Endpoint == "" { + return nil, errors.New("S3_ENDPOINT is required") + } + if cfg.Bucket == "" { + return nil, errors.New("S3_BUCKET is required") + } + if cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, errors.New("S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required") + } + + client, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + Region: cfg.Region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, fmt.Errorf("create minio client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + exists, err := client.BucketExists(ctx, cfg.Bucket) + if err != nil { + return nil, fmt.Errorf("check bucket exists: %w", err) + } + + if !exists { + err = client.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{Region: cfg.Region}) + if err != nil { + return nil, fmt.Errorf("create bucket %s: %w", cfg.Bucket, err) + } + log.Printf("INFO: Successfully created bucket '%s'", cfg.Bucket) + } + + return &Client{ + client: client, + bucket: cfg.Bucket, + cfg: cfg, + }, nil +} + +func (c *Client) Config() Config { + return c.cfg +} + +func (c *Client) PutObject(ctx context.Context, key string, data []byte, contentType string) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + _, err := c.client.PutObject( + ctx, + c.bucket, + key, + bytes.NewReader(data), + int64(len(data)), + minio.PutObjectOptions{ + ContentType: contentType, + }, + ) + if err != nil { + return fmt.Errorf("%w: put object: %v", ErrDependencyUnavailable, err) + } + + return nil +} + +type Object struct { + Body io.ReadCloser + Size int64 + ContentType string +} + +func (c *Client) GetObject(ctx context.Context, key string) (*Object, error) { + // ВАЖНО: не создаём новый контекст с таймаутом и не делаем defer cancel(). + // Контекст HTTP-запроса должен жить, пока клиент читает поток (io.Copy). + // Он будет отменён автоматически при отключении клиента. + + info, err := c.client.StatObject(ctx, c.bucket, key, minio.StatObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.StatusCode == 404 || errResp.Code == "NoSuchKey" || errResp.Code == "NotFound" { + return nil, ErrObjectNotFound + } + return nil, fmt.Errorf("%w: stat object: %v", ErrDependencyUnavailable, err) + } + + obj, err := c.client.GetObject(ctx, c.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("%w: get object: %v", ErrDependencyUnavailable, err) + } + + contentType := info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + + return &Object{ + Body: obj, + Size: info.Size, + ContentType: contentType, + }, nil +} + +func (c *Client) DeleteObject(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + err := c.client.RemoveObject(ctx, c.bucket, key, minio.RemoveObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + + if errResp.StatusCode == 404 || + errResp.Code == "NoSuchKey" || + errResp.Code == "NotFound" { + return nil + } + + return fmt.Errorf("%w: delete object: %v", ErrDependencyUnavailable, err) + } + + return nil +} + +func getenv(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func getInt64(key string, fallback int64) int64 { + v := os.Getenv(key) + if v == "" { + return fallback + } + + parsed, err := strconv.ParseInt(v, 10, 64) + if err != nil || parsed <= 0 { + return fallback + } + + return parsed +} diff --git a/src/internal/store/migrations/001_initial.sql b/src/internal/store/migrations/001_initial.sql new file mode 100644 index 00000000..f06a5643 --- /dev/null +++ b/src/internal/store/migrations/001_initial.sql @@ -0,0 +1,47 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') + ); + +CREATE INDEX idx_users_username ON users(username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX idx_chat_members_user ON chat_members(user_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL CHECK (length(text) >= 1 AND length(text) <= 4000), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_messages_chat_created ON messages(chat_id, created_at, id); + +CREATE TABLE sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash VARCHAR(64) NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_sessions_token ON sessions(token_hash); +CREATE INDEX idx_sessions_user ON sessions(user_id); +CREATE INDEX idx_sessions_expires ON sessions(expires_at); \ No newline at end of file diff --git a/src/internal/store/migrations/002_idempotency.sql b/src/internal/store/migrations/002_idempotency.sql new file mode 100644 index 00000000..2031428d --- /dev/null +++ b/src/internal/store/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, -- SHA-256 hex canonical JSON body + response_code INT NOT NULL, + response_body JSONB NOT NULL, -- сохранённый Message для реплея + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + INTERVAL '24 hours', + UNIQUE (user_id, method, path, key) + ); + +CREATE INDEX IF NOT EXISTS idx_idem_expires ON idempotency_keys (expires_at); + +-- Сообщения связываем с ключом, чтобы при реплее вернуть тот же объект +ALTER TABLE messages + ADD COLUMN IF NOT EXISTS idempotency_key_id UUID REFERENCES idempotency_keys(id) ON DELETE SET NULL; \ No newline at end of file diff --git a/src/internal/store/migrations/003_attachments.sql b/src/internal/store/migrations/003_attachments.sql new file mode 100644 index 00000000..4563093e --- /dev/null +++ b/src/internal/store/migrations/003_attachments.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + status TEXT NOT NULL DEFAULT 'ready' + CHECK (status IN ('uploading', 'ready', 'failed')), + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat_id ON attachments (chat_id); +CREATE INDEX IF NOT EXISTS idx_attachments_status_created_at ON attachments (status, created_at); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + position INT NOT NULL, + PRIMARY KEY (message_id, attachment_id) + ); + +CREATE INDEX IF NOT EXISTS idx_message_attachments_message_id ON message_attachments (message_id, position); diff --git a/src/internal/store/migrations/004_allow_empty_text.sql b/src/internal/store/migrations/004_allow_empty_text.sql new file mode 100644 index 00000000..aebb4e1b --- /dev/null +++ b/src/internal/store/migrations/004_allow_empty_text.sql @@ -0,0 +1,3 @@ +-- Лабораторная 4: разрешаем пустой текст, если есть вложения. +-- Валидация "непустой текст ИЛИ непустой attachment_ids" выполняется на уровне приложения (handler.go). +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check; \ No newline at end of file diff --git a/src/internal/store/migrations/005_async_jobs.sql b/src/internal/store/migrations/005_async_jobs.sql new file mode 100644 index 00000000..13583693 --- /dev/null +++ b/src/internal/store/migrations/005_async_jobs.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS attachment_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'published', 'processing', 'done', 'failed')), + attempts INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 5, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_status_next_run + ON attachment_jobs (status, next_run_at); \ No newline at end of file diff --git a/src/internal/store/migrations/006_update_attachment_status.sql b/src/internal/store/migrations/006_update_attachment_status.sql new file mode 100644 index 00000000..af37111b --- /dev/null +++ b/src/internal/store/migrations/006_update_attachment_status.sql @@ -0,0 +1,43 @@ +-- Лабораторная 5: асинхронная обработка вложений +-- 1. Расширяем статусы вложений: добавляем 'deleting' для промежуточного состояния при очистке +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'deleting')); + +-- 2. Таблица заданий на обработку (обратная связь между БД и брокером) +CREATE TABLE IF NOT EXISTS attachment_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'published', 'processing', 'done', 'failed')), + attempts INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 5, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending + ON attachment_jobs (status, next_run_at) + WHERE status IN ('pending', 'published'); + +-- 3. Таблица вариантов вложения (оригинал, 256×256, 128×128, WebP optimized) +-- Используется в уровне 4 лабы 5 и далее +CREATE TABLE IF NOT EXISTS attachment_variants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + kind TEXT NOT NULL + CHECK (kind IN ('original', 'thumbnail_256', 'crop_128', 'optimized_webp')), + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + sha256 CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (attachment_id, kind) + ); + +CREATE INDEX IF NOT EXISTS idx_attachment_variants_attachment + ON attachment_variants (attachment_id); \ No newline at end of file diff --git a/src/internal/store/migrations/007_async_jobs.sql b/src/internal/store/migrations/007_async_jobs.sql new file mode 100644 index 00000000..a1d10fe8 --- /dev/null +++ b/src/internal/store/migrations/007_async_jobs.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS attachment_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'published', 'processing', 'done', 'failed')), + attempts INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 5, + next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_status_next_run + ON attachment_jobs (status, next_run_at) + WHERE status IN ('pending', 'published'); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_attachment_id + ON attachment_jobs (attachment_id); \ No newline at end of file diff --git a/src/internal/store/migrations/008_attachment_variants.sql b/src/internal/store/migrations/008_attachment_variants.sql new file mode 100644 index 00000000..12db57fb --- /dev/null +++ b/src/internal/store/migrations/008_attachment_variants.sql @@ -0,0 +1,18 @@ +-- Лабораторная 5: хранение вариантов вложения (оригинал + 256×256) +CREATE TABLE IF NOT EXISTS attachment_variants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + kind TEXT NOT NULL + CHECK (kind IN ('original', 'thumbnail_256', 'crop_128', 'optimized_webp')), + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + sha256 CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (attachment_id, kind) + ); + +CREATE INDEX IF NOT EXISTS idx_attachment_variants_attachment + ON attachment_variants (attachment_id); \ No newline at end of file diff --git a/src/internal/store/migrations/009_variant_contract.sql b/src/internal/store/migrations/009_variant_contract.sql new file mode 100644 index 00000000..c59e2096 --- /dev/null +++ b/src/internal/store/migrations/009_variant_contract.sql @@ -0,0 +1,10 @@ +DELETE FROM attachment_variants +WHERE kind = 'original' OR width < 1 OR height < 1 OR size_bytes < 1; + +ALTER TABLE attachment_variants DROP CONSTRAINT IF EXISTS attachment_variants_kind_check; +ALTER TABLE attachment_variants ADD CONSTRAINT attachment_variants_kind_check + CHECK (kind IN ('thumbnail_256', 'crop_128', 'optimized_webp')); + +ALTER TABLE attachment_variants DROP CONSTRAINT IF EXISTS attachment_variants_dims_check; +ALTER TABLE attachment_variants ADD CONSTRAINT attachment_variants_dims_check + CHECK (width >= 1 AND height >= 1 AND size_bytes >= 1); \ No newline at end of file diff --git a/src/internal/store/migrations/010_level4.sql b/src/internal/store/migrations/010_level4.sql new file mode 100644 index 00000000..4192b273 --- /dev/null +++ b/src/internal/store/migrations/010_level4.sql @@ -0,0 +1,27 @@ +-- Lease и версия pipeline +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS leased_by TEXT; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease + ON attachment_jobs (lease_until) WHERE status = 'processing'; + +-- staging хранится отдельно от финального ключа +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +UPDATE attachments SET staging_key = object_key +WHERE staging_key IS NULL AND status <> 'ready'; + +-- варианты: только контрактные имена, размеры >= 1 +DELETE FROM attachment_variants +WHERE kind NOT IN ('thumbnail_256','crop_128','optimized_webp') + OR width < 1 OR height < 1 OR size_bytes < 1; + +ALTER TABLE attachment_variants DROP CONSTRAINT IF EXISTS attachment_variants_kind_check; +ALTER TABLE attachment_variants ADD CONSTRAINT attachment_variants_kind_check + CHECK (kind IN ('thumbnail_256','crop_128','optimized_webp')); + +ALTER TABLE attachment_variants DROP CONSTRAINT IF EXISTS attachment_variants_dims_check; +ALTER TABLE attachment_variants ADD CONSTRAINT attachment_variants_dims_check + CHECK (width >= 1 AND height >= 1 AND size_bytes >= 1); + +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; \ No newline at end of file diff --git a/src/internal/store/migrations/011_fix_jobs_schema.sql b/src/internal/store/migrations/011_fix_jobs_schema.sql new file mode 100644 index 00000000..6254612b --- /dev/null +++ b/src/internal/store/migrations/011_fix_jobs_schema.sql @@ -0,0 +1,59 @@ +-- Восстановление схемы attachment_jobs. +-- Причина: дубль номера миграции 005 → часть DDL не применилась, +-- а CREATE TABLE IF NOT EXISTS в 007 не добавляет колонки к существующей таблице. + +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS attachment_id UUID REFERENCES attachments(id) ON DELETE CASCADE; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS attempts INT NOT NULL DEFAULT 0; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS max_attempts INT NOT NULL DEFAULT 5; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS last_error TEXT; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS leased_by TEXT; +ALTER TABLE attachment_jobs ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; + +ALTER TABLE attachment_jobs DROP CONSTRAINT IF EXISTS attachment_jobs_status_check; +ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_status_check + CHECK (status IN ('pending','published','processing','done','failed')); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending + ON attachment_jobs (status, next_run_at) + WHERE status IN ('pending','published'); + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease + ON attachment_jobs (lease_until) WHERE status = 'processing'; + +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_attachment + ON attachment_jobs (attachment_id); + +-- staging-ключ и финальный ключ живут раздельно +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +UPDATE attachments SET staging_key = object_key +WHERE staging_key IS NULL AND status <> 'ready'; + +-- варианты: полный набор колонок + уникальность ключа эффекта +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS object_key TEXT; +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS content_type TEXT; +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS size_bytes BIGINT; +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS width INT; +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS height INT; +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS sha256 CHAR(64); +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ALTER TABLE attachment_variants ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; + +DELETE FROM attachment_variants +WHERE kind NOT IN ('thumbnail_256','crop_128','optimized_webp') + OR width IS NULL OR width < 1 + OR height IS NULL OR height < 1 + OR size_bytes IS NULL OR size_bytes < 1 + OR object_key IS NULL; + +-- UNIQUE нужен для ON CONFLICT (attachment_id, kind) — ключ идемпотентности эффекта +CREATE UNIQUE INDEX IF NOT EXISTS uq_attachment_variants_attachment_kind + ON attachment_variants (attachment_id, kind); + +-- индекс дедупликации, который потерялся вместе с пропущенным 005 +CREATE INDEX IF NOT EXISTS idx_attachments_sha256_status + ON attachments (sha256, status) WHERE status = 'ready'; \ No newline at end of file diff --git a/src/internal/store/migrations/012_attachment_sha256_index.sql b/src/internal/store/migrations/012_attachment_sha256_index.sql new file mode 100644 index 00000000..288be46b --- /dev/null +++ b/src/internal/store/migrations/012_attachment_sha256_index.sql @@ -0,0 +1,5 @@ +-- Ускоряет поиск существующих файлов с тем же содержимым. +CREATE INDEX IF NOT EXISTS idx_attachments_sha256_status + ON attachments (sha256, status) + WHERE status = 'ready'; + diff --git a/src/internal/store/postgres.go b/src/internal/store/postgres.go new file mode 100644 index 00000000..680cc266 --- /dev/null +++ b/src/internal/store/postgres.go @@ -0,0 +1,1273 @@ +package store + +import ( + "context" + "crypto/sha256" + "embed" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "strconv" + "strings" + "time" + "unicode/utf8" + + "mtusi2026lab5_fork/src/internal/model" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +var ( + ErrInvalidRequest = errors.New("invalid_request") + ErrNotFound = errors.New("not_found") + ErrConflict = errors.New("conflict") + ErrUnauthorized = errors.New("unauthorized") +) + +const migrationAdvisoryLockID int64 = 7272727272727272 + +type Store interface { + CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) + GetUserByID(ctx context.Context, id string) (*model.User, error) + GetUserByUsername(ctx context.Context, username string) (*model.User, error) + + CreateChat(ctx context.Context, title string, memberIDs []string) (*model.Chat, error) + GetChat(ctx context.Context, chatID, userID string) (*model.Chat, error) + ListChats(ctx context.Context, userID string, limit int, cursor string) ([]*model.Chat, *string, error) + + AddMessage(ctx context.Context, chatID, senderID, text string) (*model.Message, error) + SendMessageIdempotent(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, path, requestHash string) (*model.Message, error) + GetMessages(ctx context.Context, chatID, userID, cursor string, limit int) ([]*model.Message, *string, error) + + CreateSession(ctx context.Context, userID, tokenHash string, expiresAt time.Time) (*model.Session, error) + GetSession(ctx context.Context, tokenHash string) (*model.Session, error) + DeleteSession(ctx context.Context, tokenHash string) error + + CreateAttachmentAndJob(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, *model.Job, error) + GetAttachmentForMetadata(ctx context.Context, attachmentID, userID string) (*model.Attachment, error) + GetAttachmentForDownload(ctx context.Context, attachmentID, userID string) (*model.Attachment, error) + GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) + MarkAttachmentReadyWithVariant(ctx context.Context, attachmentID, thumbnailKey string, thumbSize int64, thumbSHA string, thumbWidth, thumbHeight int) error + MarkAttachmentFailed(ctx context.Context, attachmentID string, errorCode string) error + + PublishJobToBroker(ctx context.Context, jobID string) error + GetPendingJobs(ctx context.Context, limit int) ([]model.Job, error) + UpdateJobStatus(ctx context.Context, jobID string, status string, attempts int, lastError string) error + + GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) + GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) + + ClaimStaleQueued(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) + MarkAttachmentDeleting(ctx context.Context, id string) error + DeleteFailedStaleAttachment(ctx context.Context, id string) error + GetAttachmentVariant(ctx context.Context, attachmentID, userID, variantName string) (*model.Variant, error) + + GetJobByID(ctx context.Context, jobID string) (*model.Job, error) + ClaimJob(ctx context.Context, jobID, workerID string, lease time.Duration) (*model.Job, error) + RequeueExpiredLeases(ctx context.Context) (int64, error) + RequeueAttachment(ctx context.Context, attachmentID string) error + CompleteAttachment(ctx context.Context, attachmentID, jobID, finalKey string, variants []model.Variant) error + FailJob(ctx context.Context, jobID, attachmentID, errorCode, detail string) error + RetryJob(ctx context.Context, jobID, attachmentID, errorCode, detail string, delay time.Duration) error + ListCompletedStaging(ctx context.Context, olderThan time.Time) ([]model.StagingRef, error) + ClearStagingKey(ctx context.Context, attachmentID string) error +} + +type PostgresStore struct { + pool *pgxpool.Pool +} + +func NewPostgres(connString string) (*PostgresStore, error) { + config, err := pgxpool.ParseConfig(connString) + if err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + config.MaxConns = 25 + config.MinConns = 5 + config.MaxConnLifetime = 30 * time.Minute + config.MaxConnIdleTime = 5 * time.Minute + config.HealthCheckPeriod = 30 * time.Second + + pool, err := pgxpool.NewWithConfig(context.Background(), config) + if err != nil { + return nil, fmt.Errorf("create pool: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + + store := &PostgresStore{pool: pool} + if err := store.applyMigrations(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("apply migrations: %w", err) + } + + return store, nil +} + +func (s *PostgresStore) Close() { + s.pool.Close() +} + +func (s *PostgresStore) applyMigrations(ctx context.Context) error { + conn, err := s.pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire conn for lock: %w", err) + } + defer conn.Release() + + if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock($1)`, migrationAdvisoryLockID); err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + defer func() { + _, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, migrationAdvisoryLockID) + }() + + _, err = conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())`) + if err != nil { + return fmt.Errorf("create migrations table: %w", err) + } + + var lastVersion int + err = conn.QueryRow(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&lastVersion) + if err != nil { + return fmt.Errorf("get last version: %w", err) + } + + entries, err := migrationsFS.ReadDir("migrations") + if err != nil { + return fmt.Errorf("read migrations: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + var version int + if _, err := fmt.Sscanf(entry.Name(), "%d_", &version); err != nil { + continue + } + if version <= lastVersion { + log.Printf("INFO: Skipping migration %s (already applied)", entry.Name()) + continue + } + content, err := migrationsFS.ReadFile("migrations/" + entry.Name()) + if err != nil { + return fmt.Errorf("read migration %s: %w", entry.Name(), err) + } + if _, err := conn.Exec(ctx, string(content)); err != nil { + return fmt.Errorf("execute migration %s: %w", entry.Name(), err) + } + if _, err := conn.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, version); err != nil { + return fmt.Errorf("insert migration version: %w", err) + } + log.Printf("INFO: Applied migration %s", entry.Name()) + } + return nil +} + +func (s *PostgresStore) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) { + var user model.User + err := s.pool.QueryRow(ctx, `INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3) RETURNING id, username, display_name, created_at`, username, displayName, passwordHash). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + if err != nil { + if isUniqueViolation(err) { + return nil, ErrConflict + } + return nil, fmt.Errorf("insert user: %w", err) + } + user.CreatedAt = user.CreatedAt.UTC() + return &user, nil +} + +func (s *PostgresStore) GetUserByID(ctx context.Context, id string) (*model.User, error) { + var user model.User + err := s.pool.QueryRow(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1`, id). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.PasswordHash, &user.CreatedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get user by id: %w", err) + } + user.CreatedAt = user.CreatedAt.UTC() + return &user, nil +} + +func (s *PostgresStore) GetUserByUsername(ctx context.Context, username string) (*model.User, error) { + var user model.User + err := s.pool.QueryRow(ctx, `SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1`, username). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.PasswordHash, &user.CreatedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get user by username: %w", err) + } + user.CreatedAt = user.CreatedAt.UTC() + return &user, nil +} + +func (s *PostgresStore) CreateChat(ctx context.Context, title string, memberIDs []string) (*model.Chat, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + var chat model.Chat + if err := tx.QueryRow(ctx, `INSERT INTO chats (title) VALUES ($1) RETURNING id, created_at`, title).Scan(&chat.ID, &chat.CreatedAt); err != nil { + return nil, fmt.Errorf("insert chat: %w", err) + } + for _, userID := range memberIDs { + if _, err := tx.Exec(ctx, `INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)`, chat.ID, userID); err != nil { + return nil, fmt.Errorf("insert member %s: %w", userID, err) + } + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + + chat.Title = title + chat.MemberIDs = memberIDs + if chat.MemberIDs == nil { + chat.MemberIDs = []string{} + } + chat.CreatedAt = chat.CreatedAt.UTC() + return &chat, nil +} + +func (s *PostgresStore) GetChat(ctx context.Context, chatID, userID string) (*model.Chat, error) { + var isMember bool + if err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, chatID, userID).Scan(&isMember); err != nil { + return nil, fmt.Errorf("check membership: %w", err) + } + if !isMember { + return nil, ErrNotFound + } + + var chat model.Chat + if err := s.pool.QueryRow(ctx, `SELECT id, title, created_at FROM chats WHERE id = $1`, chatID).Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get chat: %w", err) + } + chat.CreatedAt = chat.CreatedAt.UTC() + chat.MemberIDs = []string{} + + rows, err := s.pool.Query(ctx, `SELECT user_id FROM chat_members WHERE chat_id = $1`, chatID) + if err != nil { + return nil, fmt.Errorf("get members: %w", err) + } + defer rows.Close() + for rows.Next() { + var m string + if err := rows.Scan(&m); err != nil { + return nil, fmt.Errorf("scan member: %w", err) + } + chat.MemberIDs = append(chat.MemberIDs, m) + } + return &chat, nil +} + +func (s *PostgresStore) ListChats(ctx context.Context, userID string, limit int, cursor string) ([]*model.Chat, *string, error) { + fetchLimit := limit + 1 + var query string + var args []interface{} + + if cursor == "" { + query = `SELECT c.id, c.title, c.created_at FROM chats c INNER JOIN chat_members cm ON cm.chat_id = c.id WHERE cm.user_id = $1 ORDER BY c.created_at, c.id LIMIT $2` + args = []interface{}{userID, fetchLimit} + } else { + query = ` + SELECT c.id, c.title, c.created_at + FROM chats c + INNER JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1 + AND (c.created_at, c.id) > ( + SELECT c2.created_at, c2.id FROM chats c2 WHERE c2.id = $2 + ) + ORDER BY c.created_at, c.id + LIMIT $3 + ` + args = []interface{}{userID, cursor, fetchLimit} + } + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, nil, fmt.Errorf("query chats: %w", err) + } + defer rows.Close() + + var chats []*model.Chat + for rows.Next() { + var chat model.Chat + if err := rows.Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil { + return nil, nil, fmt.Errorf("scan chat: %w", err) + } + chat.CreatedAt = chat.CreatedAt.UTC() + chat.MemberIDs = []string{} + chats = append(chats, &chat) + } + + if len(chats) > 0 { + ids := make([]string, len(chats)) + for i, c := range chats { + ids[i] = c.ID + } + mrows, err := s.pool.Query(ctx, `SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY($1::uuid[])`, ids) + if err != nil { + return nil, nil, fmt.Errorf("query members: %w", err) + } + defer mrows.Close() + mm := make(map[string][]string) + for mrows.Next() { + var cid, mid string + if err := mrows.Scan(&cid, &mid); err != nil { + return nil, nil, fmt.Errorf("scan member: %w", err) + } + mm[cid] = append(mm[cid], mid) + } + for _, c := range chats { + if members, ok := mm[c.ID]; ok { + c.MemberIDs = members + } + } + } + + var nextCursor *string + if len(chats) > limit { + chats = chats[:limit] + lastID := chats[len(chats)-1].ID + nextCursor = &lastID + } + if chats == nil { + chats = []*model.Chat{} + } + return chats, nextCursor, nil +} + +func (s *PostgresStore) AddMessage(ctx context.Context, chatID, senderID, text string) (*model.Message, error) { + return s.insertMessage(ctx, chatID, senderID, text) +} + +func (s *PostgresStore) SendMessageIdempotent(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, path, requestHash string) (*model.Message, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + if len(attachmentIDs) > 10 { + return nil, ErrInvalidRequest + } + if utf8.RuneCountInString(text) > 4000 { + return nil, ErrInvalidRequest + } + if strings.TrimSpace(text) == "" && len(attachmentIDs) == 0 { + return nil, ErrInvalidRequest + } + + seen := make(map[string]struct{}, len(attachmentIDs)) + for _, id := range attachmentIDs { + if _, exists := seen[id]; exists { + return nil, ErrInvalidRequest + } + seen[id] = struct{}{} + } + + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + scope := fmt.Sprintf("%s|%s|%s|%s", senderID, "POST", path, idemKey) + lockID := hashScope(scope) + + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, lockID); err != nil { + return nil, fmt.Errorf("advisory xact lock: %w", err) + } + + var existingHash string + var existingBody []byte + err = tx.QueryRow(ctx, `SELECT request_hash, response_body FROM idempotency_keys WHERE user_id = $1 AND method = $2 AND path = $3 AND key = $4`, senderID, "POST", path, idemKey).Scan(&existingHash, &existingBody) + + switch { + case err == nil: + if existingHash != requestHash { + return nil, ErrConflict + } + var msg model.Message + if err := json.Unmarshal(existingBody, &msg); err != nil { + return nil, fmt.Errorf("decode replay body: %w", err) + } + if msg.AttachmentIDs == nil { + msg.AttachmentIDs = []string{} + } + msg.CreatedAt = msg.CreatedAt.UTC() + return &msg, nil + + case errors.Is(err, pgx.ErrNoRows): + // Lab 5: обработка вложения асинхронная (queued → processing → ready). + // Сообщение принимается сразу после 202 Accepted, не дожидаясь worker. + // Отклоняем только терминальные плохие состояния: failed, deleting. + for _, attID := range attachmentIDs { + var attChatID string + var status string + qerr := tx.QueryRow(ctx, `SELECT chat_id::text, status FROM attachments WHERE id = $1`, attID).Scan(&attChatID, &status) + if qerr != nil { + if errors.Is(qerr, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if isUUIDSyntaxError(qerr) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("select attachment: %w", qerr) + } + if attChatID != chatID { + return nil, ErrNotFound + } + switch status { + case "queued", "processing", "ready": + // допустимо: вложение принадлежит чату и не в плохом состоянии + default: + return nil, ErrConflict + } + } + + msg, err := s.insertMessageTx(ctx, tx, chatID, senderID, text) + if err != nil { + return nil, err + } + msg.AttachmentIDs = attachmentIDs + + for i, attID := range attachmentIDs { + if _, err := tx.Exec(ctx, `INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1, $2, $3)`, msg.ID, attID, i); err != nil { + if isUniqueViolation(err) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("insert message attachment: %w", err) + } + } + + body, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("marshal message: %w", err) + } + if _, err := tx.Exec(ctx, `INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, response_code, response_body) VALUES ($1, $2, $3, $4, $5, $6, $7)`, senderID, "POST", path, idemKey, requestHash, 201, body); err != nil { + return nil, fmt.Errorf("insert idempotency key: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit tx: %w", err) + } + return msg, nil + + default: + return nil, fmt.Errorf("select idempotency key: %w", err) + } +} + +func (s *PostgresStore) insertMessage(ctx context.Context, chatID, senderID, text string) (*model.Message, error) { + var isMember bool + if err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, chatID, senderID).Scan(&isMember); err != nil { + return nil, fmt.Errorf("check membership: %w", err) + } + if !isMember { + return nil, ErrNotFound + } + var msg model.Message + if err := s.pool.QueryRow(ctx, `INSERT INTO messages (chat_id, sender_id, text) VALUES ($1, $2, $3) RETURNING id, chat_id, sender_id, text, created_at`, chatID, senderID, text).Scan(&msg.ID, &msg.ChatID, &msg.SenderID, &msg.Text, &msg.CreatedAt); err != nil { + return nil, fmt.Errorf("insert message: %w", err) + } + msg.CreatedAt = msg.CreatedAt.UTC() + msg.AttachmentIDs = []string{} + return &msg, nil +} + +func (s *PostgresStore) insertMessageTx(ctx context.Context, tx pgx.Tx, chatID, senderID, text string) (*model.Message, error) { + var isMember bool + if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, chatID, senderID).Scan(&isMember); err != nil { + return nil, fmt.Errorf("check membership: %w", err) + } + if !isMember { + return nil, ErrNotFound + } + var msg model.Message + if err := tx.QueryRow(ctx, `INSERT INTO messages (chat_id, sender_id, text) VALUES ($1, $2, $3) RETURNING id, chat_id, sender_id, text, created_at`, chatID, senderID, text).Scan(&msg.ID, &msg.ChatID, &msg.SenderID, &msg.Text, &msg.CreatedAt); err != nil { + return nil, fmt.Errorf("insert message: %w", err) + } + msg.CreatedAt = msg.CreatedAt.UTC() + msg.AttachmentIDs = []string{} + return &msg, nil +} + +func (s *PostgresStore) GetMessages(ctx context.Context, chatID, userID, cursor string, limit int) ([]*model.Message, *string, error) { + var isMember bool + if err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, chatID, userID).Scan(&isMember); err != nil { + return nil, nil, fmt.Errorf("check membership: %w", err) + } + if !isMember { + return nil, nil, ErrNotFound + } + + fetchLimit := limit + 1 + var query string + var args []interface{} + + if cursor == "" { + query = `SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT $2` + args = []interface{}{chatID, fetchLimit} + } else { + query = ` + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND (created_at, id) > ( + SELECT created_at, id FROM messages WHERE id = $2 AND chat_id = $1 + ) + ORDER BY created_at, id + LIMIT $3 + ` + args = []interface{}{chatID, cursor, fetchLimit} + } + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, nil, fmt.Errorf("query messages: %w", err) + } + defer rows.Close() + + var messages []*model.Message + for rows.Next() { + var msg model.Message + if err := rows.Scan(&msg.ID, &msg.ChatID, &msg.SenderID, &msg.Text, &msg.CreatedAt); err != nil { + return nil, nil, fmt.Errorf("scan message: %w", err) + } + msg.CreatedAt = msg.CreatedAt.UTC() + msg.AttachmentIDs = []string{} + messages = append(messages, &msg) + } + + var nextCursor *string + if len(messages) > limit { + messages = messages[:limit] + lastID := messages[len(messages)-1].ID + nextCursor = &lastID + } + if messages == nil { + messages = []*model.Message{} + } + + if len(messages) > 0 { + ids := make([]string, len(messages)) + for i, m := range messages { + ids[i] = m.ID + } + arows, err := s.pool.Query(ctx, `SELECT message_id::text, attachment_id::text FROM message_attachments WHERE message_id = ANY($1::uuid[]) ORDER BY message_id, position`, ids) + if err != nil { + return nil, nil, fmt.Errorf("query message attachments: %w", err) + } + defer arows.Close() + + byMessage := make(map[string][]string) + for arows.Next() { + var msgID, attID string + if err := arows.Scan(&msgID, &attID); err != nil { + return nil, nil, fmt.Errorf("scan message attachment: %w", err) + } + byMessage[msgID] = append(byMessage[msgID], attID) + } + if err := arows.Err(); err != nil { + return nil, nil, fmt.Errorf("message attachments rows: %w", err) + } + for _, m := range messages { + if atts, ok := byMessage[m.ID]; ok && len(atts) > 0 { + m.AttachmentIDs = atts + } else { + m.AttachmentIDs = []string{} + } + } + } + return messages, nextCursor, nil +} + +func (s *PostgresStore) CreateSession(ctx context.Context, userID, tokenHash string, expiresAt time.Time) (*model.Session, error) { + var session model.Session + if err := s.pool.QueryRow(ctx, `INSERT INTO sessions (user_id, token_hash, expires_at) VALUES ($1, $2, $3) RETURNING id, user_id, token_hash, expires_at, created_at`, userID, tokenHash, expiresAt).Scan(&session.ID, &session.UserID, &session.TokenHash, &session.ExpiresAt, &session.CreatedAt); err != nil { + return nil, fmt.Errorf("insert session: %w", err) + } + session.ExpiresAt = session.ExpiresAt.UTC() + session.CreatedAt = session.CreatedAt.UTC() + return &session, nil +} + +func (s *PostgresStore) GetSession(ctx context.Context, tokenHash string) (*model.Session, error) { + var session model.Session + if err := s.pool.QueryRow(ctx, `SELECT id, user_id, token_hash, expires_at, created_at FROM sessions WHERE token_hash = $1 AND expires_at > NOW()`, tokenHash).Scan(&session.ID, &session.UserID, &session.TokenHash, &session.ExpiresAt, &session.CreatedAt); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get session: %w", err) + } + session.ExpiresAt = session.ExpiresAt.UTC() + session.CreatedAt = session.CreatedAt.UTC() + return &session, nil +} + +func (s *PostgresStore) DeleteSession(ctx context.Context, tokenHash string) error { + if _, err := s.pool.Exec(ctx, `DELETE FROM sessions WHERE token_hash = $1`, tokenHash); err != nil { + return fmt.Errorf("delete session: %w", err) + } + return nil +} + +// max_attempts берётся из конфигурации, а не зашит в SQL. +func (s *PostgresStore) CreateAttachmentAndJob(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, *model.Job, error) { + maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5) + + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, nil, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + var att model.Attachment + err = tx.QueryRow(ctx, ` + INSERT INTO attachments (chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, status) + VALUES ($1,$2,$3,$4,$4,$5,$6,$7,'queued') + RETURNING id::text, chat_id::text, owner_id::text, filename, object_key, content_type, size_bytes, sha256, status, error_code, created_at, updated_at + `, chatID, ownerID, filename, objectKey, contentType, sizeBytes, sha256Hex).Scan( + &att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ObjectKey, &att.ContentType, + &att.SizeBytes, &att.SHA256, &att.Status, &att.ErrorCode, &att.CreatedAt, &att.UpdatedAt, + ) + if err != nil { + return nil, nil, fmt.Errorf("insert attachment: %w", err) + } + + var job model.Job + err = tx.QueryRow(ctx, ` + INSERT INTO attachment_jobs (attachment_id, status, attempts, max_attempts, next_run_at, pipeline_version) + VALUES ($1,'pending',0,$2,NOW(),1) + RETURNING id::text, attachment_id::text, status, attempts, max_attempts, next_run_at, last_error, created_at, lease_until, leased_by, pipeline_version + `, att.ID, maxAttempts).Scan( + &job.ID, &job.AttachmentID, &job.Status, &job.Attempts, &job.MaxAttempts, + &job.NextRunAt, &job.LastError, &job.CreatedAt, &job.LeaseUntil, &job.LeasedBy, &job.PipelineVersion, + ) + if err != nil { + return nil, nil, fmt.Errorf("insert job: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, nil, fmt.Errorf("commit tx: %w", err) + } + + att.CreatedAt, att.UpdatedAt = att.CreatedAt.UTC(), att.UpdatedAt.UTC() + att.Variants = []any{} + job.CreatedAt, job.NextRunAt = job.CreatedAt.UTC(), job.NextRunAt.UTC() + return &att, &job, nil +} + +// 'deleting' — внутреннее состояние, наружу отдаём контрактный enum. +func (s *PostgresStore) GetAttachmentForMetadata(ctx context.Context, attachmentID, userID string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.object_key, + a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at + FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE a.id = $1 AND cm.user_id = $2 + `, attachmentID, userID).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if isUUIDSyntaxError(err) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("get attachment for metadata: %w", err) + } + + if a.Status == "deleting" { + a.Status = "failed" + code := "stale_cleanup" + a.ErrorCode = &code + } + + variants, err := s.loadVariants(ctx, a.ID) + if err != nil { + return nil, err + } + a.CreatedAt, a.UpdatedAt = a.CreatedAt.UTC(), a.UpdatedAt.UTC() + a.Variants = variants + return &a, nil +} + +func (s *PostgresStore) GetAttachmentVariant(ctx context.Context, attachmentID, userID, variantName string) (*model.Variant, error) { + kind := variantNameToKind(variantName) + if kind == "" { + return nil, ErrInvalidRequest + } + + var v model.Variant + var gotKind string + err := s.pool.QueryRow(ctx, ` + SELECT v.kind, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256 + FROM attachment_variants v + JOIN attachments a ON a.id = v.attachment_id + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE v.attachment_id = $1 + AND cm.user_id = $2 + AND a.status = 'ready' + AND v.kind = $3 + `, attachmentID, userID, kind).Scan( + &gotKind, &v.ObjectKey, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if isUUIDSyntaxError(err) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("get attachment variant: %w", err) + } + + v.Name = variantKindToName(gotKind) + return &v, nil +} + +func (s *PostgresStore) GetAttachmentForDownload(ctx context.Context, attachmentID, userID string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.object_key, + a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, + a.created_at, a.updated_at + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE a.id = $1 AND cm.user_id = $2 AND a.status = 'ready' + `, attachmentID, userID).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if isUUIDSyntaxError(err) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("get attachment for download: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, + created_at, updated_at + FROM attachments + WHERE id = $1 + `, id).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get attachment by id: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) MarkAttachmentReadyWithVariant(ctx context.Context, attachmentID, thumbnailKey string, thumbSize int64, thumbSHA string, thumbWidth, thumbHeight int) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, ` + INSERT INTO attachment_variants + (attachment_id, kind, object_key, content_type, size_bytes, width, height, sha256) + VALUES ($1, 'thumbnail_256', $2, 'image/jpeg', $3, $4, $5, $6) + ON CONFLICT (attachment_id, kind) DO UPDATE SET + object_key = EXCLUDED.object_key, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256 + `, attachmentID, thumbnailKey, thumbSize, thumbWidth, thumbHeight, thumbSHA) + if err != nil { + return fmt.Errorf("upsert thumbnail variant: %w", err) + } + + // Статус ready выставляем только после записи всех обещанных вариантов. + _, err = tx.Exec(ctx, ` + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = NOW() + WHERE id = $1 AND status IN ('queued', 'processing') + `, attachmentID) + if err != nil { + return fmt.Errorf("update attachment status: %w", err) + } + + return tx.Commit(ctx) +} + +func (s *PostgresStore) MarkAttachmentFailed(ctx context.Context, attachmentID string, errorCode string) error { + _, err := s.pool.Exec(ctx, `UPDATE attachments SET status = 'failed', error_code = $1, updated_at = NOW() WHERE id = $2 AND status IN ('queued', 'processing')`, errorCode, attachmentID) + if err != nil { + return fmt.Errorf("mark attachment failed: %w", err) + } + return nil +} + +func (s *PostgresStore) PublishJobToBroker(ctx context.Context, jobID string) error { + _, err := s.pool.Exec(ctx, `UPDATE attachment_jobs SET status = 'published' WHERE id = $1 AND status = 'pending'`, jobID) + return err +} + +func (s *PostgresStore) GetPendingJobs(ctx context.Context, limit int) ([]model.Job, error) { + rows, err := s.pool.Query(ctx, ` + SELECT id::text, attachment_id::text, status, attempts, max_attempts, next_run_at, last_error, created_at + FROM attachment_jobs + WHERE status IN ('pending', 'published') AND next_run_at <= NOW() + ORDER BY next_run_at ASC + LIMIT $1 + `, limit) + if err != nil { + return nil, fmt.Errorf("query pending jobs: %w", err) + } + defer rows.Close() + + var jobs []model.Job + for rows.Next() { + var j model.Job + if err := rows.Scan(&j.ID, &j.AttachmentID, &j.Status, &j.Attempts, &j.MaxAttempts, &j.NextRunAt, &j.LastError, &j.CreatedAt); err != nil { + return nil, fmt.Errorf("scan job: %w", err) + } + j.CreatedAt = j.CreatedAt.UTC() + j.NextRunAt = j.NextRunAt.UTC() + jobs = append(jobs, j) + } + return jobs, rows.Err() +} + +func (s *PostgresStore) UpdateJobStatus(ctx context.Context, jobID string, status string, attempts int, lastError string) error { + var nextRun time.Time + if status == "retry" { + // Экспоненциальный backoff: 10s, 20s, 40s, 60s (cap) + delay := time.Duration(10*(1< 60*time.Second { + delay = 60 * time.Second + } + nextRun = time.Now().Add(delay) + status = "pending" + } else { + nextRun = time.Now() + } + _, err := s.pool.Exec(ctx, ` + UPDATE attachment_jobs + SET status = $1, attempts = $2, last_error = $3, next_run_at = $4, updated_at = NOW() + WHERE id = $5 + `, status, attempts, lastError, nextRun, jobID) + return err +} + +func (s *PostgresStore) GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, + created_at, updated_at + FROM attachments + WHERE sha256 = $1 AND status = 'ready' + ORDER BY created_at + LIMIT 1 + `, sha256Hex).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get ready attachment by sha256: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, + created_at, updated_at + FROM attachments + WHERE chat_id = $1 AND sha256 = $2 AND status = 'ready' + ORDER BY created_at + LIMIT 1 + `, chatID, sha256Hex).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get attachment by sha256 in chat: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) ClaimStaleQueued(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) { + rows, err := s.pool.Query(ctx, ` + UPDATE attachments SET status='failed', error_code='stale_cleanup', updated_at=NOW() + WHERE status='queued' AND created_at < $1 + RETURNING id::text, object_key + `, olderThan) + if err != nil { + return nil, fmt.Errorf("claim stale queued: %w", err) + } + defer rows.Close() + out := []model.Attachment{} + for rows.Next() { + var a model.Attachment + if err := rows.Scan(&a.ID, &a.ObjectKey); err != nil { + return nil, fmt.Errorf("scan claimed attachment: %w", err) + } + out = append(out, a) + } + return out, rows.Err() +} + +func (s *PostgresStore) MarkAttachmentDeleting(ctx context.Context, id string) error { + _, err := s.pool.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = NOW() WHERE id = $1 AND status IN ('queued', 'failed')`, id) + return err +} + +func (s *PostgresStore) DeleteFailedStaleAttachment(ctx context.Context, id string) error { + _, err := s.pool.Exec(ctx, + `DELETE FROM attachments WHERE id=$1 AND status='failed' AND error_code='stale_cleanup'`, id) + return err +} + +func hashScope(s string) int64 { + h := sha256.Sum256([]byte(s)) + return int64(binary.BigEndian.Uint64(h[:8])) +} + +func isUniqueViolation(err error) bool { + return err != nil && (contains(err.Error(), "23505") || contains(err.Error(), "duplicate key") || contains(err.Error(), "unique constraint")) +} + +func isForeignKeyViolation(err error) bool { + return err != nil && (contains(err.Error(), "23503") || contains(err.Error(), "violates foreign key")) +} + +func isUUIDSyntaxError(err error) bool { + return err != nil && contains(err.Error(), "invalid input syntax for type uuid") +} + +func contains(s, substr string) bool { + for i := 0; i+len(substr) <= len(s); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +func variantKindToName(kind string) string { + switch kind { + case "thumbnail_256": + return "thumbnail" + case "crop_128": + return "crop" + case "optimized_webp": + return "optimized" + } + return "" +} + +func variantNameToKind(name string) string { + switch name { + case "thumbnail": + return "thumbnail_256" + case "crop": + return "crop_128" + case "optimized": + return "optimized_webp" + } + return "" +} + +func (s *PostgresStore) loadVariants(ctx context.Context, attachmentID string) ([]any, error) { + rows, err := s.pool.Query(ctx, ` + SELECT kind, object_key, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = $1 + AND kind IN ('thumbnail_256', 'crop_128', 'optimized_webp') + ORDER BY kind + `, attachmentID) + if err != nil { + return nil, fmt.Errorf("query variants: %w", err) + } + defer rows.Close() + + out := []any{} + for rows.Next() { + var v model.Variant + var kind string + if err := rows.Scan(&kind, &v.ObjectKey, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256); err != nil { + return nil, fmt.Errorf("scan variant: %w", err) + } + v.Name = variantKindToName(kind) + if v.Name == "" { + continue + } + out = append(out, v) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("variants rows: %w", err) + } + return out, nil +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return def +} + +const jobCols = `id::text, attachment_id::text, status, attempts, max_attempts, + next_run_at, last_error, created_at, lease_until, leased_by, pipeline_version` + +func scanJob(row pgx.Row) (*model.Job, error) { + var j model.Job + err := row.Scan(&j.ID, &j.AttachmentID, &j.Status, &j.Attempts, &j.MaxAttempts, + &j.NextRunAt, &j.LastError, &j.CreatedAt, &j.LeaseUntil, &j.LeasedBy, &j.PipelineVersion) + if err != nil { + return nil, err + } + j.CreatedAt, j.NextRunAt = j.CreatedAt.UTC(), j.NextRunAt.UTC() + return &j, nil +} + +func (s *PostgresStore) GetJobByID(ctx context.Context, jobID string) (*model.Job, error) { + j, err := scanJob(s.pool.QueryRow(ctx, + `SELECT `+jobCols+` FROM attachment_jobs WHERE id=$1`, jobID)) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if isUUIDSyntaxError(err) { + return nil, ErrInvalidRequest + } + return nil, fmt.Errorf("get job by id: %w", err) + } + return j, nil +} + +// ClaimJob — единственный вход в processing. Атомарно: проверка статуса, +// инкремент попытки и установка lease. Два worker не получат одну работу. +func (s *PostgresStore) ClaimJob(ctx context.Context, jobID, workerID string, lease time.Duration) (*model.Job, error) { + secs := int(lease.Seconds()) + if secs < 1 { + secs = 60 + } + j, err := scanJob(s.pool.QueryRow(ctx, ` + UPDATE attachment_jobs + SET status='processing', attempts=attempts+1, leased_by=$2, + lease_until=NOW()+($3::int * INTERVAL '1 second'), updated_at=NOW() + WHERE id=$1 + AND (status IN ('pending','published') + OR (status='processing' AND lease_until IS NOT NULL AND lease_until < NOW())) + RETURNING `+jobCols, jobID, workerID, secs)) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrConflict + } + return nil, fmt.Errorf("claim job: %w", err) + } + return j, nil +} + +func (s *PostgresStore) RequeueExpiredLeases(ctx context.Context) (int64, error) { + tag, err := s.pool.Exec(ctx, ` + UPDATE attachment_jobs + SET status='pending', lease_until=NULL, leased_by=NULL, next_run_at=NOW(), updated_at=NOW() + WHERE status='processing' AND lease_until IS NOT NULL AND lease_until < NOW() + `) + if err != nil { + return 0, fmt.Errorf("requeue expired leases: %w", err) + } + return tag.RowsAffected(), nil +} + +func (s *PostgresStore) RequeueAttachment(ctx context.Context, attachmentID string) error { + _, err := s.pool.Exec(ctx, ` + UPDATE attachments SET status='queued', error_code=NULL, updated_at=NOW() + WHERE id=$1 AND status='processing' + `, attachmentID) + return err +} + +// CompleteAttachment публикует результат атомарно: варианты + final key + ready + job done. +// Готовый результат не откатывается повторной доставкой. +func (s *PostgresStore) CompleteAttachment(ctx context.Context, attachmentID, jobID, finalKey string, variants []model.Variant) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback(ctx) + + var status string + var pv int + if err := tx.QueryRow(ctx, + `SELECT status, 1 FROM attachments WHERE id=$1 FOR UPDATE`, attachmentID).Scan(&status, &pv); err != nil { + return fmt.Errorf("lock attachment: %w", err) + } + + if status != "ready" { + for _, v := range variants { + kind := variantNameToKind(v.Name) + if kind == "" { + return fmt.Errorf("unknown variant name %q", v.Name) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO attachment_variants + (attachment_id, kind, object_key, content_type, size_bytes, width, height, sha256, pipeline_version) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,1) + ON CONFLICT (attachment_id, kind) DO UPDATE SET + object_key=EXCLUDED.object_key, content_type=EXCLUDED.content_type, + size_bytes=EXCLUDED.size_bytes, width=EXCLUDED.width, + height=EXCLUDED.height, sha256=EXCLUDED.sha256 + `, attachmentID, kind, v.ObjectKey, v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256); err != nil { + return fmt.Errorf("upsert variant %s: %w", kind, err) + } + } + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET object_key=$2, status='ready', error_code=NULL, updated_at=NOW() + WHERE id=$1 + `, attachmentID, finalKey); err != nil { + return fmt.Errorf("publish final key: %w", err) + } + } + + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status='done', lease_until=NULL, leased_by=NULL, last_error=NULL, updated_at=NOW() + WHERE id=$1 + `, jobID); err != nil { + return fmt.Errorf("mark job done: %w", err) + } + return tx.Commit(ctx) +} + +func (s *PostgresStore) FailJob(ctx context.Context, jobID, attachmentID, errorCode, detail string) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status='failed', error_code=$2, updated_at=NOW() + WHERE id=$1 AND status <> 'ready' + `, attachmentID, errorCode); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status='failed', last_error=$2, lease_until=NULL, leased_by=NULL, updated_at=NOW() + WHERE id=$1 + `, jobID, detail); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (s *PostgresStore) RetryJob(ctx context.Context, jobID, attachmentID, errorCode, detail string, delay time.Duration) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + UPDATE attachments SET status='queued', error_code=NULL, updated_at=NOW() + WHERE id=$1 AND status <> 'ready' + `, attachmentID); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + UPDATE attachment_jobs + SET status='pending', last_error=$2, lease_until=NULL, leased_by=NULL, + next_run_at=NOW()+($3::bigint * INTERVAL '1 millisecond'), updated_at=NOW() + WHERE id=$1 + `, jobID, errorCode+": "+detail, int64(delay.Milliseconds())); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (s *PostgresStore) ListCompletedStaging(ctx context.Context, olderThan time.Time) ([]model.StagingRef, error) { + rows, err := s.pool.Query(ctx, ` + SELECT id::text, staging_key FROM attachments + WHERE status='ready' AND staging_key IS NOT NULL AND updated_at < $1 + LIMIT 100 + `, olderThan) + if err != nil { + return nil, err + } + defer rows.Close() + out := []model.StagingRef{} + for rows.Next() { + var r model.StagingRef + if err := rows.Scan(&r.ID, &r.StagingKey); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *PostgresStore) ClearStagingKey(ctx context.Context, attachmentID string) error { + _, err := s.pool.Exec(ctx, `UPDATE attachments SET staging_key=NULL WHERE id=$1`, attachmentID) + return err +} diff --git a/student-tests/check-download.py b/student-tests/check-download.py new file mode 100644 index 00000000..993bff8a --- /dev/null +++ b/student-tests/check-download.py @@ -0,0 +1,57 @@ +import requests +import json +import time +import base64 +import hashlib + +BASE = "http://localhost:8080" +USERNAME = f"test{int(time.time())}" +PASSWORD = "CorrectHorseBattery1" + +print("1. Creating user...") +r = requests.post(f"{BASE}/api/v1/users", json={"username": USERNAME, "display_name": "Test", "password": PASSWORD}) +print(" Status:", r.status_code) + +print("2. Logging in...") +auth = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode() +r = requests.post(f"{BASE}/api/v1/auth/sessions", headers={"Authorization": f"Basic {auth}"}) +token = r.json()["token"] +headers = {"Authorization": f"Bearer {token}"} + +print("3. Creating chat...") +r = requests.get(f"{BASE}/api/v1/users/me", headers=headers) +uid = r.json()["id"] +r = requests.post(f"{BASE}/api/v1/chats", headers=headers, json={"title": "test", "member_ids": [uid]}) +chat_id = r.json()["id"] + +print("4. Uploading attachment...") +with open("../tests/fixtures/sample.png", "rb") as f: + original_data = f.read() + +r = requests.post(f"{BASE}/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={**headers, "Content-Type": "image/png"}, + data=original_data) +att_id = r.json()["id"] +print(" Attachment ID:", att_id) + +print("5. Downloading attachment...") +# Увеличиваем таймаут, чтобы исключить преждевременное отключение +r = requests.get(f"{BASE}/api/v1/attachments/{att_id}/content", headers=headers, timeout=30) +print(" Download Status:", r.status_code) +print(" Content-Type:", r.headers.get("Content-Type")) +print(" Content-Length header:", r.headers.get("Content-Length")) + +downloaded_data = r.content +print(" Original length:", len(original_data)) +print(" Downloaded length:", len(downloaded_data)) + +orig_hash = hashlib.sha256(original_data).hexdigest() +down_hash = hashlib.sha256(downloaded_data).hexdigest() +print(" Original SHA256: ", orig_hash) +print(" Downloaded SHA256:", down_hash) + +if original_data == downloaded_data: + print("\n✅ SUCCESS: Files match perfectly!") +else: + print("\n❌ FAIL: Files differ!") + print(" First 100 bytes of downloaded data:", downloaded_data[:100]) diff --git a/student-tests/debug_smoke.py b/student-tests/debug_smoke.py new file mode 100644 index 00000000..84d0f1d6 --- /dev/null +++ b/student-tests/debug_smoke.py @@ -0,0 +1,60 @@ +import requests +import json +import time +import base64 + +BASE = "http://localhost:8080" +USERNAME = f"test{int(time.time())}" +PASSWORD = "CorrectHorseBattery1" + +def do_req(method, path, **kwargs): + url = f"{BASE}{path}" + r = requests.request(method, url, **kwargs) + print(f"=== {method} {path} (Status: {r.status_code}) ===") + try: + print(json.dumps(r.json(), indent=2)) + except: + print(r.text) + print() + return r + +# 1. Create User +r = do_req("POST", "/api/v1/users", json={"username": USERNAME, "display_name": "Test", "password": PASSWORD}) +assert r.status_code == 201 + +# 2. Login +auth = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode() +r = do_req("POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {auth}"}) +assert r.status_code == 201 +token = r.json()["token"] +headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + +# 3. Get Me +r = do_req("GET", "/api/v1/users/me", headers=headers) +uid = r.json()["id"] + +# 4. Create Chat +r = do_req("POST", "/api/v1/chats", headers=headers, json={"title": "test", "member_ids": [uid]}) +assert r.status_code == 201 +chat_id = r.json()["id"] + +# 5. Upload Attachment +with open("../tests/fixtures/sample.png", "rb") as f: + r = requests.post(f"{BASE}/api/v1/chats/{chat_id}/attachments?filename=test.png", + headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"}, + data=f.read()) +print("=== POST /attachments (Status: {r.status_code}) ===") +print(json.dumps(r.json(), indent=2)) +print() +att_id = r.json()["id"] + +# 6. Get Attachment Metadata +r = do_req("GET", f"/api/v1/attachments/{att_id}", headers={"Authorization": f"Bearer {token}"}) + +# 7. Send Message with Attachment +r = do_req("POST", f"/api/v1/chats/{chat_id}/messages", + headers={**headers, "Idempotency-Key": "test-key-1"}, + json={"text": "hello", "attachment_ids": [att_id]}) + +# 8. Get Messages +r = do_req("GET", f"/api/v1/chats/{chat_id}/messages", headers=headers) diff --git a/student-tests/fault_before_ack.sh b/student-tests/fault_before_ack.sh new file mode 100755 index 00000000..a42eb203 --- /dev/null +++ b/student-tests/fault_before_ack.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +P="python3 student-tests/lab5_probe.py" +DC="docker compose --env-file .env" +POINT="${1:-after_commit_before_ack}" +SQL() { $DC exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -c \"$1\""; } + +{ + echo "=== C. детерминированный обрыв: $POINT ===" + $P new + echo "--- пересоздаём worker с точкой обрыва (lease=5s, reclaim=3s) ---" + JOB_LEASE_SECONDS=5 JOB_POLL_SECONDS=2 JOB_RECLAIM_IDLE_SECONDS=3 FAULT_EXIT_AT="$POINT" \ + $DC up -d --force-recreate worker + $P upload + + echo "--- ждём FAULT ---" + for i in $(seq 1 60); do + if $DC logs worker --tail=200 | grep -q "FAULT:.*$POINT"; then break; fi + sleep 1 + done + $DC logs worker --tail=20 + + echo "--- состояние СРАЗУ после обрыва ---" + SQL "SELECT id,status,error_code FROM attachments ORDER BY created_at DESC LIMIT 1;" + SQL "SELECT id,status,attempts,lease_until FROM attachment_jobs ORDER BY created_at DESC LIMIT 1;" + SQL "SELECT count(*) AS variants_written FROM attachment_variants;" + echo "--- неподтверждённые сообщения (PEL) ---" + $DC exec -T broker redis-cli -n 1 XPENDING attachment_jobs worker_group - + 10 + + echo "--- перезапуск того же контейнера: marker блокирует повторный обрыв ---" + JOB_LEASE_SECONDS=5 JOB_POLL_SECONDS=2 JOB_RECLAIM_IDLE_SECONDS=3 FAULT_EXIT_AT="$POINT" \ + $DC start worker + sleep 25 + $DC logs worker --tail=25 + + $P wait --timeout 120 + echo "--- набор вариантов не удвоился ---" + SQL "SELECT attachment_id,kind,count(*) FROM attachment_variants GROUP BY 1,2 HAVING count(*)>1;" + echo "--- staging очищен ---" + SQL "SELECT count(*) AS leftover_staging FROM attachments WHERE staging_key IS NOT NULL AND status='ready';" + echo "--- PEL пуст ---" + $DC exec -T broker redis-cli -n 1 XPENDING attachment_jobs worker_group - + 10 + $P fetch --variant thumbnail + + echo "--- возврат worker в штатный режим ---" + $DC up -d --force-recreate worker +} 2>&1 | tee "evidence/C_${POINT}.txt" \ No newline at end of file diff --git a/student-tests/fault_commit_publish.sh b/student-tests/fault_commit_publish.sh new file mode 100755 index 00000000..e4b68708 --- /dev/null +++ b/student-tests/fault_commit_publish.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +P="python3 student-tests/lab5_probe.py" +DC="docker compose --env-file .env" + +{ + echo "=== B. crash между COMMIT и publish ===" + $DC stop worker broker + $P new + echo "--- upload: publish не пройдёт, намерение в БД (ожидаем 202) ---" + $P upload + echo "--- jobs в БД ---" + $DC exec -T postgres sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT id,status,attempts FROM attachment_jobs ORDER BY created_at DESC LIMIT 3;"' + echo "--- убиваем API до восстановления брокера ---" + $DC kill api + $DC start broker + sleep 8 + $DC start api worker + sleep 25 + echo "--- задача доходит до ready без ручного создания задания ---" + $P wait --timeout 150 + $P fetch --variant thumbnail +} 2>&1 | tee evidence/B_commit_publish.txt \ No newline at end of file diff --git a/student-tests/fault_lease.sh b/student-tests/fault_lease.sh new file mode 100755 index 00000000..7b3f228d --- /dev/null +++ b/student-tests/fault_lease.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +P="python3 student-tests/lab5_probe.py" +DC="docker compose --env-file .env" +SQL() { $DC exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -c \"$1\""; } +ARM="JOB_LEASE_SECONDS=5 JOB_POLL_SECONDS=2 JOB_RECLAIM_IDLE_SECONDS=3 FAULT_EXIT_AT=after_claim" + +{ + echo "=== D. смерть worker, удерживающего lease ===" + $P new + echo "--- worker с обрывом сразу после ClaimJob (lease=5s) ---" + env $ARM $DC up -d --force-recreate worker + $P upload + + for i in $(seq 1 60); do + if $DC logs worker --tail=200 | grep -q "FAULT:.*after_claim"; then break; fi + sleep 1 + done + $DC logs worker --tail=15 + + echo "--- job ЗАХВАЧЕН: processing, attempts=1, lease_until в будущем ---" + SQL "SELECT id,status,attempts,leased_by,lease_until FROM attachment_jobs ORDER BY created_at DESC LIMIT 1;" + SQL "SELECT id,status FROM attachments ORDER BY created_at DESC LIMIT 1;" + + echo "--- перезапуск того же контейнера: marker блокирует повторный обрыв ---" + env $ARM $DC start worker + sleep 20 + $DC logs worker --tail=20 + + echo "--- lease истёк, работа возвращена и завершена (attempts=2) ---" + SQL "SELECT id,status,attempts,lease_until FROM attachment_jobs ORDER BY created_at DESC LIMIT 1;" + $P wait --timeout 120 + SQL "SELECT attachment_id,kind,count(*) FROM attachment_variants GROUP BY 1,2 HAVING count(*)>1;" + $P fetch --variant crop + + echo "--- возврат worker в штатный режим ---" + $DC up -d --force-recreate worker +} 2>&1 | tee evidence/D_lease.txt \ No newline at end of file diff --git a/student-tests/fault_retry_replay.sh b/student-tests/fault_retry_replay.sh new file mode 100755 index 00000000..a7ebe1bb --- /dev/null +++ b/student-tests/fault_retry_replay.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +P="python3 student-tests/lab5_probe.py" +DC="docker compose --env-file .env" +SQL() { $DC exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -c \"$1\""; } + +{ + echo "=== E. transient-ошибка → backoff → исчерпание → DLQ → replay ===" + $P new + echo "--- max_attempts=2 (env читает API), worker падает во временную ошибку ---" + JOB_MAX_ATTEMPTS=2 JOB_POLL_SECONDS=2 FAULT_FAIL_MODE=transient FAULT_FAIL_TIMES=99 \ + $DC up -d --force-recreate api worker + sleep 15 + $P upload + + echo "--- ждём терминального failed ---" + for i in $(seq 1 60); do + s=$(SQL "SELECT status FROM attachments ORDER BY created_at DESC LIMIT 1;" | sed -n 3p | tr -d ' ') + [ "$s" = "failed" ] && break + sleep 2 + done + $DC logs worker --tail=40 + echo "--- видны две попытки с разным backoff (jitter) и error_code ---" + SQL "SELECT id,status,attempts,max_attempts,next_run_at,last_error FROM attachment_jobs ORDER BY created_at DESC LIMIT 1;" + SQL "SELECT id,status,error_code FROM attachments ORDER BY created_at DESC LIMIT 1;" + echo "--- DLQ ---" + $DC exec -T broker redis-cli -n 1 XRANGE attachment_jobs_dlq - + COUNT 10 + + echo "--- снимаем fault: worker пересоздан без инъекции ---" + JOB_MAX_ATTEMPTS=5 $DC up -d --force-recreate api worker + sleep 12 + + echo "--- операторский replay: только failed, где attachment НЕ ready ---" + SQL "WITH cand AS (SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.status='failed' AND a.status<>'ready') UPDATE attachment_jobs j SET status='pending',attempts=0,last_error=NULL,lease_until=NULL,leased_by=NULL,next_run_at=NOW(),updated_at=NOW() FROM cand WHERE j.id=cand.id RETURNING j.id,j.attachment_id;" + SQL "UPDATE attachments a SET status='queued',error_code=NULL,updated_at=NOW() FROM attachment_jobs j WHERE j.attachment_id=a.id AND j.status='pending' AND a.status='failed' RETURNING a.id;" + + $P wait --timeout 120 + SQL "SELECT attachment_id,kind,count(*) FROM attachment_variants GROUP BY 1,2 HAVING count(*)>1;" + $P fetch --variant thumbnail + + echo "--- защита от дубля: повторный replay уже ready-задачи ничего не меняет ---" + SQL "WITH cand AS (SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.status='failed' AND a.status<>'ready') UPDATE attachment_jobs j SET status='pending',attempts=0 FROM cand WHERE j.id=cand.id RETURNING j.id;" +} 2>&1 | tee evidence/E_retry_replay.txt diff --git a/student-tests/fault_worker_down.sh b/student-tests/fault_worker_down.sh new file mode 100755 index 00000000..5697d60f --- /dev/null +++ b/student-tests/fault_worker_down.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +P="python3 student-tests/lab5_probe.py" +DC="docker compose --env-file .env" + +{ + echo "=== A. worker остановлен ===" + $DC stop worker + $P new + $P upload + echo "--- content до обработки (ожидаем 409) ---" + $P fetch || true + echo "--- сообщение с queued вложением (ожидаем 201) ---" + $P send --text "queued attachment" + echo "--- рестарт API с сохранением volumes ---" + $DC restart api + sleep 20 + echo "--- рестарт broker с сохранением volume ---" + $DC restart broker + sleep 8 + echo "--- запуск worker ---" + $DC start worker + sleep 5 + $P wait --timeout 120 + echo "--- оригинал ---" + $P fetch + echo "--- thumbnail ---" + $P fetch --variant thumbnail + echo "--- crop ---" + $P fetch --variant crop + echo "--- optimized ---" + $P fetch --variant optimized +} 2>&1 | tee evidence/A_worker_down.txt \ No newline at end of file diff --git a/student-tests/lab5_probe.py b/student-tests/lab5_probe.py new file mode 100755 index 00000000..9f454974 --- /dev/null +++ b/student-tests/lab5_probe.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Синтетические проверки lab5. Токен хранится во временном файле и не печатается.""" +import argparse, base64, hashlib, json, os, sys, time, uuid +import requests + +BASE = os.environ.get("BASE_URL", "http://localhost:8080") +SESS = "/tmp/lab5_session.json" +FIXTURE = "tests/fixtures/sample.png" + + +def save(d): + with open(SESS, "w") as f: + json.dump(d, f) + + +def load(): + with open(SESS) as f: + return json.load(f) + + +def hdr(s): + return {"Authorization": "Bearer " + s["token"]} + + +def cmd_new(_): + u = "p" + uuid.uuid4().hex[:12] + p = "CorrectHorseBattery1" + r = requests.post(BASE + "/api/v1/users", + json={"username": u, "display_name": "Probe", "password": p}, timeout=15) + r.raise_for_status() + uid = r.json()["id"] + basic = base64.b64encode(f"{u}:{p}".encode()).decode() + r = requests.post(BASE + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + basic}, timeout=15) + r.raise_for_status() + token = r.json()["token"] + r = requests.post(BASE + "/api/v1/chats", headers=hdr({"token": token}), + json={"title": "probe", "member_ids": [uid]}, timeout=15) + r.raise_for_status() + save({"token": token, "user_id": uid, "chat_id": r.json()["id"]}) + print(json.dumps({"user_id": uid, "chat_id": r.json()["id"]}, indent=2)) + + +def cmd_upload(a): + s = load() + data = open(a.file or FIXTURE, "rb").read() + r = requests.post(f"{BASE}/api/v1/chats/{s['chat_id']}/attachments", + params={"filename": os.path.basename(a.file or FIXTURE)}, + headers={**hdr(s), "Content-Type": "image/png"}, data=data, timeout=60) + out = {"http_status": r.status_code, "attachment": r.json(), + "original_sha256": hashlib.sha256(data).hexdigest()} + if r.status_code == 202: + s["attachment_id"] = r.json()["id"] + s["original_sha256"] = out["original_sha256"] + save(s) + print(json.dumps(out, indent=2)) + + +def cmd_wait(a): + s = load() + aid = a.id or s["attachment_id"] + deadline = time.time() + a.timeout + last = None + while time.time() < deadline: + r = requests.get(f"{BASE}/api/v1/attachments/{aid}", headers=hdr(s), timeout=15) + r.raise_for_status() + last = r.json() + if last["status"] in ("ready", "failed"): + print(json.dumps({"elapsed_note": "terminal", "attachment": last}, indent=2)) + return 0 if last["status"] == "ready" else 1 + time.sleep(1) + print(json.dumps({"elapsed_note": "timeout", "attachment": last}, indent=2)) + return 1 + + +def cmd_fetch(a): + s = load() + aid = a.id or s["attachment_id"] + params = {"variant": a.variant} if a.variant else {} + r = requests.get(f"{BASE}/api/v1/attachments/{aid}/content", + headers=hdr(s), params=params, timeout=60) + print(json.dumps({ + "http_status": r.status_code, + "content_type": r.headers.get("Content-Type"), + "content_length_header": r.headers.get("Content-Length"), + "bytes": len(r.content), + "sha256": hashlib.sha256(r.content).hexdigest(), + }, indent=2)) + return 0 if r.status_code == 200 else 1 + + +def cmd_send(a): + s = load() + aid = a.id or s["attachment_id"] + r = requests.post(f"{BASE}/api/v1/chats/{s['chat_id']}/messages", + headers={**hdr(s), "Content-Type": "application/json", + "Idempotency-Key": uuid.uuid4().hex}, + json={"text": a.text, "attachment_ids": [aid]}, timeout=30) + print(json.dumps({"http_status": r.status_code, "message": r.json()}, indent=2)) + return 0 if r.status_code == 201 else 1 + + +def cmd_parallel(a): + import concurrent.futures as cf + s = load() + data = open(FIXTURE, "rb").read() + + def one(i): + r = requests.post(f"{BASE}/api/v1/chats/{s['chat_id']}/attachments", + params={"filename": f"par{i}.png"}, + headers={**hdr(s), "Content-Type": "image/png"}, data=data, timeout=60) + return r.status_code, r.json().get("id") + + t0 = time.time() + with cf.ThreadPoolExecutor(max_workers=a.n) as ex: + res = list(ex.map(one, range(a.n))) + ids = [x[1] for x in res if x[0] == 202] + ready = 0 + deadline = time.time() + a.timeout + while time.time() < deadline and ready < len(ids): + ready = 0 + for aid in ids: + rr = requests.get(f"{BASE}/api/v1/attachments/{aid}", headers=hdr(s), timeout=15) + if rr.json().get("status") == "ready": + ready += 1 + if ready < len(ids): + time.sleep(2) + print(json.dumps({"submitted": a.n, "accepted_202": len(ids), + "ready": ready, "wall_seconds": round(time.time() - t0, 2)}, indent=2)) + return 0 if ready == len(ids) else 1 + + +def main(): + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("new").set_defaults(fn=cmd_new) + p = sub.add_parser("upload"); p.add_argument("--file"); p.set_defaults(fn=cmd_upload) + p = sub.add_parser("wait"); p.add_argument("--id"); p.add_argument("--timeout", type=int, default=90); p.set_defaults(fn=cmd_wait) + p = sub.add_parser("fetch"); p.add_argument("--id"); p.add_argument("--variant"); p.set_defaults(fn=cmd_fetch) + p = sub.add_parser("send"); p.add_argument("--id"); p.add_argument("--text", default=""); p.set_defaults(fn=cmd_send) + p = sub.add_parser("parallel"); p.add_argument("-n", type=int, default=10); p.add_argument("--timeout", type=int, default=180); p.set_defaults(fn=cmd_parallel) + a = ap.parse_args() + sys.exit(a.fn(a) or 0) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/load_parallel.sh b/student-tests/load_parallel.sh new file mode 100755 index 00000000..ff5c9b67 --- /dev/null +++ b/student-tests/load_parallel.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +{ + python3 student-tests/lab5_probe.py new + python3 student-tests/lab5_probe.py parallel -n 12 --timeout 240 + docker stats --no-stream +} 2>&1 | tee evidence/F_parallel.txt + diff --git a/student-tests/replay_failed.sh b/student-tests/replay_failed.sh new file mode 100755 index 00000000..813de75e --- /dev/null +++ b/student-tests/replay_failed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p evidence +DC="docker compose --env-file .env" + +{ + echo "=== E. реестр failed и DLQ ===" + $DC exec -T postgres sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT id,attachment_id,status,attempts,last_error FROM attachment_jobs WHERE status='"'"'failed'"'"';"' + $DC exec -T broker redis-cli -n 1 XRANGE attachment_jobs_dlq - + COUNT 20 + + echo "=== replay: только failed, у которых attachment НЕ ready ===" + $DC exec -T postgres sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c " + WITH cand AS ( + SELECT j.id FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = '"'"'failed'"'"' AND a.status <> '"'"'ready'"'"' + ) + UPDATE attachment_jobs j + SET status='"'"'pending'"'"', attempts=0, last_error=NULL, + lease_until=NULL, leased_by=NULL, next_run_at=NOW(), updated_at=NOW() + FROM cand WHERE j.id = cand.id + RETURNING j.id, j.attachment_id;"' + $DC exec -T postgres sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c " + UPDATE attachments a SET status='"'"'queued'"'"', error_code=NULL, updated_at=NOW() + FROM attachment_jobs j + WHERE j.attachment_id=a.id AND j.status='"'"'pending'"'"' AND a.status='"'"'failed'"'"' + RETURNING a.id;"' +} 2>&1 | tee evidence/E_replay.txt \ No newline at end of file diff --git a/student-tests/run-test.py b/student-tests/run-test.py new file mode 100644 index 00000000..84d0f1d6 --- /dev/null +++ b/student-tests/run-test.py @@ -0,0 +1,60 @@ +import requests +import json +import time +import base64 + +BASE = "http://localhost:8080" +USERNAME = f"test{int(time.time())}" +PASSWORD = "CorrectHorseBattery1" + +def do_req(method, path, **kwargs): + url = f"{BASE}{path}" + r = requests.request(method, url, **kwargs) + print(f"=== {method} {path} (Status: {r.status_code}) ===") + try: + print(json.dumps(r.json(), indent=2)) + except: + print(r.text) + print() + return r + +# 1. Create User +r = do_req("POST", "/api/v1/users", json={"username": USERNAME, "display_name": "Test", "password": PASSWORD}) +assert r.status_code == 201 + +# 2. Login +auth = base64.b64encode(f"{USERNAME}:{PASSWORD}".encode()).decode() +r = do_req("POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {auth}"}) +assert r.status_code == 201 +token = r.json()["token"] +headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + +# 3. Get Me +r = do_req("GET", "/api/v1/users/me", headers=headers) +uid = r.json()["id"] + +# 4. Create Chat +r = do_req("POST", "/api/v1/chats", headers=headers, json={"title": "test", "member_ids": [uid]}) +assert r.status_code == 201 +chat_id = r.json()["id"] + +# 5. Upload Attachment +with open("../tests/fixtures/sample.png", "rb") as f: + r = requests.post(f"{BASE}/api/v1/chats/{chat_id}/attachments?filename=test.png", + headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"}, + data=f.read()) +print("=== POST /attachments (Status: {r.status_code}) ===") +print(json.dumps(r.json(), indent=2)) +print() +att_id = r.json()["id"] + +# 6. Get Attachment Metadata +r = do_req("GET", f"/api/v1/attachments/{att_id}", headers={"Authorization": f"Bearer {token}"}) + +# 7. Send Message with Attachment +r = do_req("POST", f"/api/v1/chats/{chat_id}/messages", + headers={**headers, "Idempotency-Key": "test-key-1"}, + json={"text": "hello", "attachment_ids": [att_id]}) + +# 8. Get Messages +r = do_req("GET", f"/api/v1/chats/{chat_id}/messages", headers=headers) -- 2.54.0