ЛР 5 — БВТ2303 — Морозова — на 4 #77
+32
@@ -0,0 +1,32 @@
|
||||
# --- build stage ---
|
||||
FROM golang:1.26-alpine AS build
|
||||
|
||||
WORKDIR /src
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY cmd ./cmd
|
||||
COPY src ./src
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/outbox ./cmd/outbox && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker
|
||||
|
||||
# --- runtime stage ---
|
||||
FROM alpine:3.20
|
||||
RUN apk add --no-cache ca-certificates postgresql-client && \
|
||||
addgroup -S app && adduser -S -G app app
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /out/messenger /app/messenger
|
||||
COPY --from=build /out/worker /app/worker
|
||||
COPY --from=build /out/outbox /app/outbox
|
||||
COPY migrations /app/migrations
|
||||
COPY scripts/migrate.sh /app/migrate.sh
|
||||
RUN chmod +x /app/migrate.sh
|
||||
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["/app/messenger"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover my_tests my_tests_quick my_tests_slow
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,24 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# Быстрые (<1 мин): статусы, лимиты, варианты, payload, DLQ, staging.
|
||||
my_tests_quick:
|
||||
python3 student_tests/states.py
|
||||
python3 student_tests/limits.py
|
||||
python3 student_tests/variants.py
|
||||
python3 student_tests/broker-payload.py
|
||||
python3 student_tests/staging-cleanup.py
|
||||
python3 student_tests/dlq.py
|
||||
python3 student_tests/redelivery.py
|
||||
|
||||
# Долгие (>2 мин): retry/backoff/failed/replay/outbox/lease/crash.
|
||||
my_tests_slow:
|
||||
python3 student_tests/outbox.py
|
||||
python3 student_tests/failed.py
|
||||
python3 student_tests/retry.py
|
||||
python3 student_tests/replay.py
|
||||
python3 student_tests/lease.py
|
||||
python3 student_tests/crash-before-ack.py
|
||||
|
||||
my_tests: my_tests_quick my_tests_slow
|
||||
@@ -1,37 +1,184 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Морозова Ольга Андреевна, БВТ2303
|
||||
- Целевая оценка: **4** (треки на 5 не заявлены)
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники: [README.md](README.md), [COURSE.md](COURSE.md), [contracts/openapi.json](contracts/openapi.json), документация Go, MinIO (mc), PostgreSQL, Redis Streams, Nginx, Docker.
|
||||
- Библиотеки: Go 1.26 (stdlib), github.com/google/uuid v1.6.0, github.com/jackc/pgx/v5 v5.5.5, github.com/redis/go-redis/v9 v9.5.1, golang.org/x/crypto v0.23.0, github.com/minio/minio-go/v7 v7.0.72, golang.org/x/image v0.48.0. Тесты — Python 3 (stdlib).
|
||||
- Помощь ИИ: Deepseek — написание и проверка кода.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Команды выполнялись в WSL2 (Ubuntu) на Windows 11. Требования: Docker 24+ (Compose v2), Python 3.9+ (для тестов), `make`.
|
||||
|
||||
Версии (фиксированные теги образов):
|
||||
|
||||
- `golang:1.26-alpine` — build stage
|
||||
- `alpine:3.20` — runtime stage
|
||||
- `postgres:16.3-alpine` — база данных
|
||||
- `redis:7.2.5-alpine` — кэш сессий и брокер (Redis Streams)
|
||||
- `nginx:1.27.0-alpine` — балансировщик
|
||||
- `pgsty/silo:RELEASE.2026-10-26T00-00-00Z` — S3-совместимое хранилище (MinIO-compatible)
|
||||
|
||||
### Подготовка
|
||||
|
||||
```sh
|
||||
git clone https://gitea.privcloud.tech/MiniLynx/mtusi2026lab5-fork
|
||||
cd mtusi2026lab5-fork
|
||||
|
||||
# 1. Секреты (не входят в Git)
|
||||
mkdir -p secrets
|
||||
printf 'local-dev-postgres-password\n' > secrets/postgres_password.txt
|
||||
printf 'postgres://messenger:local-dev-postgres-password@postgres:5432/messenger?sslmode=disable\n' > secrets/database_url.txt
|
||||
printf 'redis://redis:6379/0\n' > secrets/cache_url.txt
|
||||
printf 'minioadmin\n' > secrets/minio_root_user.txt
|
||||
printf 'minioadmin-dev-password\n' > secrets/minio_root_password.txt
|
||||
printf 'messenger-api\n' > secrets/s3_access_key_id.txt
|
||||
printf 'messenger-api-dev-secret\n' > secrets/s3_secret_access_key.txt
|
||||
chmod 600 secrets/*.txt
|
||||
|
||||
# 2. Локальный .env
|
||||
cp .env.example .env
|
||||
# ВАЖНО: В .env нужно исправить S3_ENDPOINT на s3:9000 (без http://)
|
||||
sed -i 's|S3_ENDPOINT=.*|S3_ENDPOINT=s3:9000|' .env
|
||||
|
||||
# 3. Python для проверок
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
python -m pip install -r requirements-dev.txt
|
||||
|
||||
# 4. Скрипты
|
||||
chmod +x scripts/migrate.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh
|
||||
```
|
||||
|
||||
### Сборка и запуск
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Дождаться /health/ready
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
Для запуска с другим количеством реплик добавить `API_REPLICAS=$n` перед `docker compose --env-file .env up -d --build`, где `$n` - желаемое количество реплик.
|
||||
|
||||
### Тесты
|
||||
|
||||
```sh
|
||||
make check
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make test
|
||||
|
||||
# Persistence
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
|
||||
# Failover
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
./scripts/restore-api.sh
|
||||
|
||||
# Собственные тесты (pipeline, сбои, лимиты)
|
||||
make my_tests
|
||||
```
|
||||
|
||||
## Остановка (с сохранением volumes)
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down --remove-orphans
|
||||
```
|
||||
|
||||
### Намеренное удаление учебных данных
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down --remove-orphans -v
|
||||
rm -f secrets/*.txt
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
HTTP-клиент → Nginx (8080) → API (Go, x2, stateless)
|
||||
│
|
||||
├──[S3 PUT]→ staging/ (S3, приватный bucket)
|
||||
├──[INSERT]→ Postgres: attachments (queued) + attachment_jobs (outbox)
|
||||
└──[202 Accepted]
|
||||
|
||||
Outbox (x1) ⇄ Postgres (attachment_jobs.published_at IS NULL)
|
||||
│
|
||||
└──[XADD]→ Redis Streams (attachment.jobs)
|
||||
|
||||
Worker (x2) ⇄ Redis Streams (consumer group)
|
||||
├──[SELECT ... FOR UPDATE SKIP LOCKED]→ Postgres (lease 60s)
|
||||
├──[GET]→ staging/ → resize (thumb 256, crop 128, opt) →[PUT]→ final/
|
||||
├──[INSERT variants + UPDATE ready]→ Postgres (транзакция)
|
||||
└──[XACK]→ Redis
|
||||
```
|
||||
|
||||
- **Асинхронный приём (уровень 3):** `POST /api/v1/chats/{id}/attachments` возвращает **202 Accepted** сразу после того, как файл сохранён в staging, а в БД одной транзакцией записаны метаданные `attachments` (status=`queued`) и `attachment_jobs` (outbox). Это гарантирует, что после ответа 202 работа не потеряется при рестарте API.
|
||||
- **Outbox (уровень 4):** отдельный процесс `cmd/outbox` читает `attachment_jobs WHERE published_at IS NULL` и публикует в брокер. `published_at` ставится только после `XADD`. Падение между COMMIT и publish приводит к повторной публикации при следующем цикле; worker идемпотентен.
|
||||
- **Брокер:** Redis Streams (тот же Redis, AOF `--appendonly yes`). Consumer Group `workers`, `XACK` после durable-эффекта. В сообщении только `job_id`, `attachment_id`, `pipeline_version`, `attempt` (никаких URL или filename).
|
||||
- **Worker (уровень 3-4):** отдельный процесс, реплики 2. Забирает job через `ClaimJobByID` (lease 60s). Пишет `original` (server-side copy staging→final), `thumbnail` (256×256, fit), `crop` (128×128, center-cover) и `optimized` (JPEG q=70). Всё в одной транзакции с `attachment_variants` и `status='ready'`. `XACK` только после COMMIT.
|
||||
- **Идемпотентность:** ключ эффекта — `(attachment_id, pipeline_version, variant)`. Повторная доставка того же job_id: `MarkReady` — no-op (если `status='done'`), `attachment_variants` — `ON CONFLICT DO NOTHING`. Также `VariantsExist` перед Claim.
|
||||
- **Retry:** `RetryJob` различает permanent (`invalid_image` → сразу failed) и transient (`storage_error` → backoff `base * 2^(n-1) + jitter`, max 5 попыток). Финальный `failed` с безопасным `error_code`.
|
||||
- **Зависшие job:** `RequeueStuck` в outbox и worker, каждые 30с: `status='processing' AND lease_expires_at < now()` → `queued`, `published_at=NULL`.
|
||||
- **Replay:** `POST /api/v1/attachments/{id}/replay` сбрасывает `attempt=1`, `status='queued'`, `published_at=NULL` для failed-задач.
|
||||
- **Очистка:** `MoveToFinal` удаляет staging после CopyObject. `ReconcileOrphans` (в API) чистит `attachment_gc` и «зависшие» staging-объекты без job.
|
||||
|
||||
### Что перенесено из lab4
|
||||
|
||||
- PostgreSQL, Argon2id, sessions (с кэшем в Redis), горизонтальное масштабирование, Nginx.
|
||||
- Механизм идемпотентности `idempotency_keys` для сообщений.
|
||||
- S3-хранилище и приватный bucket, `S3Store` с таймаутами и ретраями.
|
||||
|
||||
### Что изменено
|
||||
|
||||
- Миграция `0002_attachments.sql`: добавлены `attachment_variants`, `attachment_jobs` (outbox+lease), поля `staging_key`, `pipeline_version`, `ready_at`, `failed_at`. Таблица `attachments` теперь содержит `status IN ('queued','processing','ready','failed')`.
|
||||
- `UploadAttachment` возвращает 202, статус `queued`; `DownloadAttachment` → 409 для не-ready.
|
||||
- Добавлены `cmd/outbox`, `cmd/worker` (обработка изображений, `process.go`).
|
||||
- `src/domain/broker.go` — обёртка Redis Streams.
|
||||
- `compose.yaml`: сервисы `outbox` (1 реплика) и `worker` (2 реплики), миграции вынесены в отдельный сервис `migrate`.
|
||||
- В `attachments.go`: `EnqueueAttachment`, `ClaimJob`, `ClaimJobByID`, `MarkReady`, `RetryJob`, `RequeueStuck`, `ReplayFailed`, `MarkFailedPermanent`, `GetVariantKey`, `SelectUnpublishedJobs`, `MarkPublished`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------ |
|
||||
| Offline-структура и локальные refs | `make check` | PASS | [png](evidence/2_Tests_Passing.png) |
|
||||
| Полная OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | [png](evidence/2_Tests_Passing.png) |
|
||||
| Публичный smoke (lab5) | `make test` | PASS: 202, ready, thumbnail | [png](evidence/2_Tests_Passing.png) |
|
||||
| Persistence: перезапуск API+БД+Redis+S3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | [png](evidence/2_Tests_Passing.png) |
|
||||
| Failover: остановка одной реплики API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | `observed_instances_before: 2`, `successful_writes: 75`, `first_recovery: 5.371s` | [png](evidence/2_Tests_Passing.png) |
|
||||
| **Ур. 3:** Upload → 202 → queued (worker не нужен) | `student_tests/states.py` | queued → ready, 3 варианта + original через `?variant=original` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** Broker и worker — отдельные процессы | `docker compose ps` | `outbox`, `worker-1`, `worker-2` | [png](evidence/1_Healthcheck.png) |
|
||||
| **Ур. 3:** Статусы в API, лимиты и типы | `student_tests/limits.py` | 413, 415, 400 invalid_image | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** original неизменен, thumbnail по контракту | `student_tests/variants.py` | `byte-identical`, `thumbnail 256x160 (metadata matches)`, `crop 128x128` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** Broker payload без filename/URL | `student_tests/broker-payload.py` | только id/version/attempt | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** Очистка staging после ready | `student_tests/staging-cleanup.py` | `staging object removed after ready` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** Реестр failed (DLQ) | `student_tests/dlq.py` | `invalid_image\|1` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 3:** Идемпотентность при повторной доставке | `student_tests/redelivery.py` | `still 3 non-original variants` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Outbox закрывает разрыв COMMIT→publish | `student_tests/outbox.py` | `published_at IS NULL` при паузе outbox → ready после unpause | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Permanent error без retry | `student_tests/failed.py` | `attempt=1, invalid_image — not retried` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Transient error → retry → ready | `student_tests/retry.py` | `attempt=2`, `retry log lines: ... backoff=2s jitter=451ms` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Replay failed | `student_tests/replay.py` | `replay -> queued`, повторный permanent fail | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Lease / reaper зависших | `student_tests/lease.py` | `reaper requeued the stuck job` → `ready` | [txt](evidence/my_tests_passing.txt) |
|
||||
| **Ур. 4:** Crash после side effect до ack | `student_tests/crash-before-ack.py` | `recovery produced exactly 3 variants, no duplicates` | [txt](evidence/my_tests_passing.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- |
|
||||
| Остановка одной реплики API | 0 потерянных подтверждённых записей, LB переключается | Failover PASS, 75 успешных сообщений | 5.371 с, потерь нет |
|
||||
| Рестарт API/БД/Redis/S3 (volumes) | Вложение и сессия живы | Persistence PASS | ~30 с (запуск контейнеров), потерь нет |
|
||||
| Отказ S3 | API возвращает 503, worker ретраит | `retry.py` PASS, `attempt=2`, backoff сработал | После восстановления S3 — ready |
|
||||
| Пауза outbox (разрыв COMMIT→publish) | `published_at=NULL`, upload 202 | `outbox.py` PASS, job дошёл до ready после unpause | Задержка publish = период паузы, потерь нет |
|
||||
| Пауза worker (зависший processing) | Lease истекает, reaper возвращает в queued | `lease.py` PASS, job вернулся в queued через ~60с + reaper | ~90 с, потерь нет |
|
||||
| `SIGKILL` worker во время processing | Повторная доставка, без дублей вариантов | `crash-before-ack.py` PASS, 3 варианта, без дублей | ~100 с (lease + рестарт), потерь нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **At-least-once:** доставка Redis Streams + идемпотентный эффект ≠ exactly-once. При падении между XADD и MarkPublished возможна повторная публикация; повторный worker это корректно обрабатывает, но сообщение в Stream продублируется.
|
||||
- **Replay** доступен через endpoint `POST /api/v1/attachments/{id}/replay` (расширение сверх OpenAPI). В отчёте зафиксировано, что это не меняет семантику обязательных путей.
|
||||
- **Optimized** — JPEG q=70 вместо WebP (WebP-кодирование требует cgo-зависимостей, которых в alpine-сборке без CGO нет). Контракт допускает JPEG как «перекодированный» вариант; в отчёте явно указано.
|
||||
- **Broker = Redis Streams** вместо RabbitMQ — обосновано тем, что Redis уже нужен для кэша сессий, AOF включён, это уменьшает количество контейнеров. Если Redis упадёт без AOF-реплики, неопубликованные задачи будут ждать в `attachment_jobs` (outbox) — они не потеряются.
|
||||
- **DLQ** — пока реестр `attachment_jobs.status='failed'` + `last_error_code`, без отдельного UI. Для оператора — `SELECT` из БД и endpoint replay.
|
||||
- **Не реализовано:** версионирование нескольких pipeline (трек 5), auto-scaling worker’а по queue lag (трек 1), fault-injection с версионированием (трек 2). Уровень 5 не заявлен.
|
||||
- **Нагрузка** проверена только smoke + `make test`; полноценных замеров throughput/latency pipeline не делалось (не требуется для уровня 4).
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/src/api"
|
||||
"mtusi-messenger/src/config"
|
||||
"mtusi-messenger/src/domain"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if cfg.DatabaseURL == "" {
|
||||
log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)")
|
||||
}
|
||||
|
||||
rootCtx, cancelRoot := context.WithCancel(context.Background())
|
||||
defer cancelRoot()
|
||||
|
||||
store, err := domain.NewStore(rootCtx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
var s3store *domain.S3Store
|
||||
if cfg.S3Endpoint != "" {
|
||||
s3store, err = domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
} else {
|
||||
log.Fatal("S3_ENDPOINT is required for lab4")
|
||||
}
|
||||
|
||||
// Кэш сессий в Redis/Valkey. Если CACHE_URL не задан — работаем без кэша.
|
||||
var cache *domain.SessionCache
|
||||
if cfg.CacheURL != "" {
|
||||
cache, err = domain.NewSessionCache(rootCtx, cfg.CacheURL)
|
||||
if err != nil {
|
||||
if cfg.RedisRequired {
|
||||
log.Fatalf("redis required but unavailable: %v", err)
|
||||
}
|
||||
log.Printf("redis unavailable, continuing without session cache: %v", err)
|
||||
cache = nil
|
||||
} else {
|
||||
defer cache.Close()
|
||||
}
|
||||
} else if cfg.RedisRequired {
|
||||
log.Fatal("SESSION_STORE=redis but CACHE_URL is not configured")
|
||||
}
|
||||
|
||||
sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL)
|
||||
handler := &api.Handler{Store: store, Sessions: sessions, S3: s3store, MaxUpload: cfg.MaxUploadBytes}
|
||||
router := api.NewRouter(handler, cfg.InstanceID)
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(2 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-rootCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := store.ReconcileOrphans(rootCtx, s3store, cfg.S3GCGrace); err != nil {
|
||||
log.Printf("reconcile: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
Handler: router,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 15 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
go func() {
|
||||
log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL)
|
||||
log.Printf("instance_id=%s", cfg.InstanceID)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("server error: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-stop
|
||||
log.Println("shutting down...")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(ctx)
|
||||
log.Println("stopped")
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/src/config"
|
||||
"mtusi-messenger/src/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
publishInterval = 1 * time.Second
|
||||
reapInterval = 30 * time.Second
|
||||
publishBatch = 50
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if cfg.DatabaseURL == "" {
|
||||
log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)")
|
||||
}
|
||||
if cfg.CacheURL == "" {
|
||||
log.Fatal("CACHE_URL is required (env CACHE_URL or CACHE_URL_FILE)")
|
||||
}
|
||||
|
||||
rootCtx, cancelRoot := context.WithCancel(context.Background())
|
||||
defer cancelRoot()
|
||||
|
||||
store, err := domain.NewStore(rootCtx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
broker, err := domain.NewBroker(rootCtx, cfg.CacheURL)
|
||||
if err != nil {
|
||||
log.Fatalf("broker: %v", err)
|
||||
}
|
||||
|
||||
s3, err := domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
|
||||
go gcLoop(rootCtx, store, s3, cfg.S3GCGrace)
|
||||
|
||||
// Reaper зависших processing крутится здесь, потому что outbox уже
|
||||
// держит соединение с БД и Redis и запускается в единственном экземпляре.
|
||||
go reaperLoop(rootCtx, store)
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
go func() {
|
||||
<-stop
|
||||
log.Println("outbox: shutting down")
|
||||
cancelRoot()
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(publishInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
log.Printf("outbox: running (interval=%s batch=%d)", publishInterval, publishBatch)
|
||||
for {
|
||||
select {
|
||||
case <-rootCtx.Done():
|
||||
log.Println("outbox: stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := publishBatchOnce(rootCtx, store, broker); err != nil {
|
||||
log.Printf("outbox: publish: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// publishBatchOnce: выбираем партию неопубликованных job-ов, публикуем
|
||||
// по одному, после XADD — MarkPublished. Падение между XADD и MarkPublished
|
||||
// приведёт к повторной публикации при следующем цикле; worker идемпотентен
|
||||
// (ключ эффекта — (attachment_id, pipeline_version, variant)), поэтому
|
||||
// дубль в Streams — норма, а не баг.
|
||||
func publishBatchOnce(ctx context.Context, store *domain.Store, broker *domain.Broker) error {
|
||||
jobs, err := store.SelectUnpublishedJobs(ctx, publishBatch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, j := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
msg := domain.JobMessage{
|
||||
SchemaVersion: 1,
|
||||
JobID: j.JobID,
|
||||
Type: "attachment.process.v1",
|
||||
AttachmentID: j.AttachmentID,
|
||||
PipelineVersion: j.PipelineVersion,
|
||||
Attempt: j.Attempt,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if _, err := broker.Publish(ctx, msg); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
// Не фатально: попробуем в следующем цикле, зафиксировав причину.
|
||||
_ = store.BumpPublishAttempt(ctx, j.JobID, "publish_failed")
|
||||
log.Printf("outbox: publish job=%s attachment=%s: %v",
|
||||
j.JobID, j.AttachmentID, err)
|
||||
continue
|
||||
}
|
||||
if err := store.MarkPublished(ctx, j.JobID); err != nil {
|
||||
log.Printf("outbox: mark published job=%s: %v", j.JobID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reaperLoop(ctx context.Context, store *domain.Store) {
|
||||
t := time.NewTicker(reapInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
n, err := store.RequeueStuck(ctx)
|
||||
if err != nil {
|
||||
log.Printf("outbox: reaper: %v", err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("outbox: reaper requeued %d stuck job(s)", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func gcLoop(ctx context.Context, store *domain.Store, s3 *domain.S3Store, grace time.Duration) {
|
||||
t := time.NewTicker(2 * time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if err := store.ReconcileOrphans(ctx, s3, grace); err != nil {
|
||||
log.Printf("outbox: gc: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/src/config"
|
||||
"mtusi-messenger/src/domain"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
lease = 60 * time.Second
|
||||
maxAttempts = 5
|
||||
baseBackoff = 2 * time.Second
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if cfg.CacheURL == "" {
|
||||
log.Fatal("CACHE_URL (redis) is required for broker")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
store, err := domain.NewStore(ctx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
s3, err := domain.NewS3Store(ctx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
|
||||
broker, err := domain.NewBroker(ctx, cfg.CacheURL)
|
||||
if err != nil {
|
||||
log.Fatalf("broker: %v", err)
|
||||
}
|
||||
|
||||
consumer := cfg.InstanceID + "-worker"
|
||||
|
||||
go reaperLoop(ctx, store)
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
go func() {
|
||||
<-stop
|
||||
log.Println("worker shutting down")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
for ctx.Err() == nil {
|
||||
msgs, err := broker.Consume(ctx, consumer)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Printf("consume: %v", err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
for _, m := range msgs {
|
||||
handleOne(ctx, store, s3, broker, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reaperLoop(ctx context.Context, store *domain.Store) {
|
||||
t := time.NewTicker(30 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if n, err := store.RequeueStuck(ctx); err != nil {
|
||||
log.Printf("reaper: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("reaper: requeued %d stuck job(s)", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store,
|
||||
broker *domain.Broker, m redis.XMessage) {
|
||||
|
||||
var job domain.JobMessage
|
||||
if raw, ok := m.Values["payload"].(string); ok {
|
||||
_ = json.Unmarshal([]byte(raw), &job)
|
||||
} else if raw, ok := m.Values["payload"].([]byte); ok {
|
||||
_ = json.Unmarshal(raw, &job)
|
||||
} else {
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Валидация схемы (обязательные поля).
|
||||
if job.SchemaVersion != 1 || job.Type != "attachment.process.v1" ||
|
||||
job.JobID == "" || job.AttachmentID == "" {
|
||||
log.Printf("poison message %s, acking", m.ID)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// No-op для уже готового результата (redelivery после готовности).
|
||||
if done, _ := store.VariantsExist(ctx, job.AttachmentID, job.PipelineVersion); done {
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
j, err := store.ClaimJobByID(ctx, job.JobID, workerID(), lease)
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
// Уже done/failed или lease держит кто-то другой — ack, не дублируем.
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("claim: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
stagingKey, _, err := store.GetStagingKey(ctx, j.AttachmentID)
|
||||
if err != nil {
|
||||
log.Printf("staging key: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
raw, ct, err := s3.FetchStagingForProcessing(ctx, stagingKey)
|
||||
if err != nil {
|
||||
code, permanent := classify(err)
|
||||
if permanent {
|
||||
// постоянная ошибка — failed без retry
|
||||
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, code)
|
||||
} else {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, code, maxAttempts, baseBackoff)
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
variants, err := processImage(raw, ct)
|
||||
if err != nil {
|
||||
// decode error — постоянная
|
||||
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, "invalid_image")
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
finalOriginalKey := "final/" + j.AttachmentID + "/original"
|
||||
var vrows []domain.Variant
|
||||
for _, v := range variants {
|
||||
if v.Name == "original" {
|
||||
// оригинал перекладываем из staging server-side.
|
||||
if err := s3.MoveToFinal(ctx, stagingKey, finalOriginalKey); err != nil {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
key := "final/" + j.AttachmentID + "/" + v.Name
|
||||
sum, err := s3.PutBytes(ctx, key, v.ContentType, v.Data)
|
||||
if err != nil {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
v.SHA256 = sum
|
||||
}
|
||||
vrows = append(vrows, domain.Variant{
|
||||
Name: v.Name, ContentType: v.ContentType, SizeBytes: int64(len(v.Data)),
|
||||
Width: v.Width, Height: v.Height, SHA256: v.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
if err := store.MarkReady(ctx, j.JobID, j.AttachmentID, j.PipelineVersion, finalOriginalKey, vrows); err != nil {
|
||||
// Уже done — не страшно; иначе retry.
|
||||
if !errors.Is(err, domain.ErrConflict) {
|
||||
log.Printf("mark ready: %v", err)
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
}
|
||||
|
||||
func workerID() string {
|
||||
h, _ := os.Hostname()
|
||||
return h + "-" + time.Now().Format("150405")
|
||||
}
|
||||
|
||||
func classify(err error) (string, bool) {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
return "payload_too_large", true
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
return "unsupported_media_type", true
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
return "invalid_image", true
|
||||
default:
|
||||
return "storage_error", false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"math"
|
||||
|
||||
"golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
type VariantSpec struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Width int
|
||||
Height int
|
||||
Data []byte
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, src, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// processImage возвращает 4 варианта. Оригинал — byte-identical с входом.
|
||||
// thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop),
|
||||
// optimized — JPEG q=70 без метаданных (WebP оставлен на усмотрение; см. отчёт).
|
||||
func processImage(raw []byte, contentType string) ([]VariantSpec, error) {
|
||||
src, _, err := image.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]VariantSpec, 0, 4)
|
||||
|
||||
// original — ровно те байты, что пришли в staging.
|
||||
sum := sha256.Sum256(raw)
|
||||
out = append(out, VariantSpec{
|
||||
Name: "original", ContentType: contentType,
|
||||
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
|
||||
Data: raw, SHA256: hex.EncodeToString(sum[:]),
|
||||
})
|
||||
|
||||
// thumbnail: fit в 256x160 с сохранением пропорций (для 16:10 — 256x160).
|
||||
th := resizeFit(src, 256, 256)
|
||||
if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// crop 128x128: центр-cover до меньшего квадрата.
|
||||
cr := squareCover(src, 128)
|
||||
if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// optimized
|
||||
optBuf, err := encodeOptimizedJPEG(src, 70)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
optSum := sha256.Sum256(optBuf)
|
||||
out = append(out, VariantSpec{
|
||||
Name: "optimized", ContentType: "image/jpeg",
|
||||
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
|
||||
Data: optBuf, SHA256: hex.EncodeToString(optSum[:]),
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func appendEncoded(out *[]VariantSpec, name, ct string, img image.Image) error {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(buf.Bytes())
|
||||
*out = append(*out, VariantSpec{
|
||||
Name: name, ContentType: ct,
|
||||
Width: img.Bounds().Dx(), Height: img.Bounds().Dy(),
|
||||
Data: buf.Bytes(), SHA256: hex.EncodeToString(sum[:]),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func resizeFit(src image.Image, maxW, maxH int) image.Image {
|
||||
b := src.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
scale := float64(maxW) / float64(w)
|
||||
if float64(maxH)/float64(h) < scale {
|
||||
scale = float64(maxH) / float64(h)
|
||||
}
|
||||
if scale > 1 {
|
||||
scale = 1
|
||||
}
|
||||
nw := int(math.Round(float64(w) * scale))
|
||||
nh := int(math.Round(float64(h) * scale))
|
||||
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// squareCover масштабирует src так, чтобы покрыть квадрат size×size
|
||||
// (min-масштаб по двум осям), затем центр-кроп до size×size.
|
||||
// Результат — всегда ровно size×size.
|
||||
func squareCover(src image.Image, size int) image.Image {
|
||||
b := src.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
// scale — больший из двух, чтобы покрыть квадрат
|
||||
scale := float64(size) / float64(w)
|
||||
if float64(size)/float64(h) > scale {
|
||||
scale = float64(size) / float64(h)
|
||||
}
|
||||
nw := int(math.Round(float64(w) * scale))
|
||||
nh := int(math.Round(float64(h) * scale))
|
||||
if nw < size {
|
||||
nw = size
|
||||
}
|
||||
if nh < size {
|
||||
nh = size
|
||||
}
|
||||
|
||||
scaled := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||
draw.CatmullRom.Scale(scaled, scaled.Bounds(), src, b, draw.Over, nil)
|
||||
|
||||
// center-crop до size×size
|
||||
x0 := (nw - size) / 2
|
||||
y0 := (nh - size) / 2
|
||||
dst := image.NewRGBA(image.Rect(0, 0, size, size))
|
||||
draw.Draw(dst, dst.Bounds(), scaled,
|
||||
image.Pt(x0, y0), draw.Src)
|
||||
return dst
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
x-api: &api
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
env_file:
|
||||
- .env
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
s3: { condition: service_healthy }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
networks: [backend]
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.3-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
secrets: [postgres_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${PG_CPUS:-1.0}", memory: "${PG_MEM:-512M}" }
|
||||
|
||||
redis:
|
||||
image: redis:7.2.5-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--save"]
|
||||
volumes: [redisdata:/data]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
|
||||
s3:
|
||||
image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
secrets: [minio_root_user, minio_root_password]
|
||||
volumes: [s3data:/data]
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEM:-512M}" }
|
||||
|
||||
s3-init:
|
||||
image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
entrypoint: ["/bin/sh", "/bootstrap/bootstrap.sh"]
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro
|
||||
- /tmp/silo-init-data:/data
|
||||
networks: [backend]
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy: { condition: "no" }
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_PER_REPLICA:-0.5}"
|
||||
memory: "${API_MEM_PER_REPLICA:-128M}"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
start_period: 20s
|
||||
retries: 5
|
||||
|
||||
outbox:
|
||||
<<: *api
|
||||
entrypoint: ["/app/outbox"]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
deploy:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits: { cpus: "0.2", memory: "64M" }
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
entrypoint: ["/app/worker"]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
s3: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: "256M" }
|
||||
|
||||
migrate:
|
||||
<<: *api
|
||||
entrypoint: ["/app/migrate.sh"]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy:
|
||||
condition: "no"
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.0-alpine
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "8080:8080"
|
||||
depends_on: [api]
|
||||
networks: [backend]
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
s3data:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
driver: bridge
|
||||
|
||||
secrets:
|
||||
postgres_password: { file: ./secrets/postgres_password.txt }
|
||||
database_url: { file: ./secrets/database_url.txt }
|
||||
cache_url: { file: ./secrets/cache_url.txt }
|
||||
s3_access_key_id: { file: ./secrets/s3_access_key_id.txt }
|
||||
s3_secret_access_key: { file: ./secrets/s3_secret_access_key.txt }
|
||||
minio_root_user: { file: ./secrets/minio_root_user.txt }
|
||||
minio_root_password: { file: ./secrets/minio_root_password.txt }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 94 KiB |
@@ -0,0 +1,56 @@
|
||||
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ make my_tests
|
||||
python3 student_tests/states.py
|
||||
upload -> queued
|
||||
status stays queued while worker paused
|
||||
ready with thumbnail/crop/optimized (original available via ?variant=original)
|
||||
original served via ?variant=original (111817 bytes)
|
||||
python3 student_tests/limits.py
|
||||
413 payload_too_large
|
||||
415 unsupported_media_type
|
||||
415 mismatched content (client MIME not trusted)
|
||||
400 invalid image (signature ok, header broken)
|
||||
400 too many pixels
|
||||
python3 student_tests/variants.py
|
||||
original byte-identical
|
||||
thumbnail 256x160 (metadata matches)
|
||||
crop 128x128 JPEG
|
||||
optimized present (4818 bytes)
|
||||
python3 student_tests/broker-payload.py
|
||||
stream sample:
|
||||
1794848167433-0
|
||||
payload
|
||||
{"schema_version":1,"job_id":"ee59551f-c85f-411a-b4d9-6076f31fde8f","type":"attachment.process.v1","attachment_id":"1175cf24-2984-4800-b84d-1aa702ecd7bd","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:07Z"}
|
||||
1794848176445-0
|
||||
payload
|
||||
{"schema_version":1,"job_id":"b7792daf-6123-4c7a-9566-b2fc4a53a614","type":"attachment.process.v1","attachment_id":"05ef10e7-19c4-47ef-9c68-a2e09eb8338e","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:16Z"}
|
||||
|
||||
broker payload has only ids/version/attempt
|
||||
python3 student_tests/staging-cleanup.py
|
||||
staging object removed after ready (mc rc=1)
|
||||
python3 student_tests/dlq.py
|
||||
failed registry:
|
||||
b1ccbeca-ae22-44f1-837c-868d2165889c|invalid_image|1
|
||||
python3 student_tests/redelivery.py
|
||||
redelivery is no-op (still 3 non-original variants; original via object_key)
|
||||
python3 student_tests/outbox.py
|
||||
outbox paused: published_at IS NULL, upload accepted
|
||||
after unpause: ready
|
||||
python3 student_tests/failed.py
|
||||
failed immediately (attempt=1), error_code=invalid_image — permanent, not retried
|
||||
python3 student_tests/retry.py
|
||||
transient error retried: attempt=2
|
||||
retry log lines:
|
||||
worker-1 | 2026/11/16 16:56:48 retry job=64072f3d-6b9e-4101-b31b-7e9a2f51bac3 attempt=2 next_attempt=2026-11-16T16:56:50.964432673Z backoff=2s jitter=451.35473ms code=storage_error
|
||||
recovered to ready after staging restored (attempt=2)
|
||||
python3 student_tests/replay.py
|
||||
failed reached
|
||||
replay -> queued
|
||||
replay reset attempt to 1; permanent error failed again — not retried
|
||||
python3 student_tests/lease.py
|
||||
worker stopped at processing (job=da0adcf7-64a7-427c-a50b-438e9d688bb7, lease_expires_at=2026-11-16 16:57:59.829609+00)
|
||||
reaper requeued the stuck job
|
||||
ready after unstuck
|
||||
python3 student_tests/crash-before-ack.py
|
||||
worker killed during processing
|
||||
recovery produced exactly 3 non-original variants, no duplicates (original in object_key)
|
||||
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$
|
||||
@@ -0,0 +1,31 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.5.5
|
||||
github.com/minio/minio-go/v7 v7.0.72
|
||||
github.com/redis/go-redis/v9 v9.5.1
|
||||
golang.org/x/crypto v0.23.0
|
||||
golang.org/x/image v0.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
golang.org/x/net v0.23.0 // indirect
|
||||
golang.org/x/sync v0.25.0 // indirect
|
||||
golang.org/x/sys v0.50.0 // indirect
|
||||
golang.org/x/text v0.44.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
|
||||
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA=
|
||||
github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
|
||||
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
|
||||
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
|
||||
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
MINIO_ROOT_USER_FILE="${MINIO_ROOT_USER_FILE:-/run/secrets/minio_root_user}"
|
||||
MINIO_ROOT_PASSWORD_FILE="${MINIO_ROOT_PASSWORD_FILE:-/run/secrets/minio_root_password}"
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}"
|
||||
S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
API_USER_FILE="${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}"
|
||||
API_PASS_FILE="${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}"
|
||||
|
||||
for f in "$MINIO_ROOT_USER_FILE" "$MINIO_ROOT_PASSWORD_FILE"; do
|
||||
[ -f "$f" ] || { echo "ERROR: missing $f" >&2; exit 1; }
|
||||
done
|
||||
|
||||
ROOT_USER="$(cat "$MINIO_ROOT_USER_FILE")"
|
||||
ROOT_PASS="$(cat "$MINIO_ROOT_PASSWORD_FILE")"
|
||||
|
||||
mc alias set local "$S3_ENDPOINT" "$ROOT_USER" "$ROOT_PASS" >/dev/null
|
||||
|
||||
# 1) bucket: приватный, создаётся один раз
|
||||
mc mb --ignore-existing "local/$S3_BUCKET" >/dev/null
|
||||
mc anonymous set none "local/$S3_BUCKET" >/dev/null
|
||||
|
||||
# 2) ограниченный пользователь API (если креды заданы заранее)
|
||||
if [ -s "$API_USER_FILE" ] && [ -s "$API_PASS_FILE" ]; then
|
||||
API_USER="$(cat "$API_USER_FILE")"
|
||||
API_PASS="$(cat "$API_PASS_FILE")"
|
||||
if ! mc admin user info local "$API_USER" >/dev/null 2>&1; then
|
||||
mc admin user add local "$API_USER" "$API_PASS" >/dev/null
|
||||
fi
|
||||
# Политика: только PutObject/GetObject/DeleteObject в bucket.
|
||||
cat >/tmp/api-policy.json <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}]
|
||||
}
|
||||
EOF
|
||||
mc admin policy create local api-attachments /tmp/api-policy.json >/dev/null 2>&1 || true
|
||||
mc admin policy attach local api-attachments --user "$API_USER" >/dev/null 2>&1 || true
|
||||
else
|
||||
echo "WARN: API S3 credentials not pre-created; generate and place them in secrets/" >&2
|
||||
fi
|
||||
|
||||
echo "s3 bootstrap done (bucket=$S3_BUCKET)"
|
||||
@@ -0,0 +1,29 @@
|
||||
worker_processes auto;
|
||||
events { worker_connections 4096; }
|
||||
|
||||
http {
|
||||
# Docker's embedded DNS. nginx будет обращаться сюда за IP реплик.
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 5s;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 12m;
|
||||
|
||||
location / {
|
||||
set $api_backend http://api:8080;
|
||||
|
||||
proxy_pass $api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 15s;
|
||||
proxy_read_timeout 15s;
|
||||
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
-- ============================================================
|
||||
-- Core: users, sessions, chats, members, messages, idempotency
|
||||
-- ============================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор.
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
status_code INTEGER NOT NULL,
|
||||
response JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_created
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,88 @@
|
||||
-- ============================================================
|
||||
-- Attachments: metadata, variants, jobs/outbox, GC
|
||||
-- ============================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0 AND size_bytes <= 10485760),
|
||||
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width INTEGER NOT NULL CHECK (width >= 0),
|
||||
height INTEGER NOT NULL CHECK (height >= 0),
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued','processing','ready','failed','pending')),
|
||||
error_code TEXT,
|
||||
-- поля из 0006
|
||||
pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
staging_key TEXT,
|
||||
ready_at TIMESTAMPTZ,
|
||||
failed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id);
|
||||
-- Для reconciler/поиска «зависших» pending-записей:
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_pending
|
||||
ON attachments (created_at) WHERE status <> 'ready';
|
||||
|
||||
-- Связь сообщение ↔ вложения. FK на элементы массива Postgres не умеет,
|
||||
-- целостность — на уровне приложения (или триггера, если нужно).
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
|
||||
|
||||
-- Журнал компенсации: объекты S3, которые могли остаться без метаданных.
|
||||
CREATE TABLE IF NOT EXISTS attachment_gc (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
reason TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Idempotency эффекта: (attachment_id, pipeline_version, variant)
|
||||
-- content_type без CHECK: варианты могут быть webp и т.п.
|
||||
-- sha256 — text+CHECK, чтобы не ловить padding от char(64).
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
variant TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
-- Outbox + очередь job-ов
|
||||
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||
job_id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued','processing','done','failed')),
|
||||
-- outbox
|
||||
published_at TIMESTAMPTZ,
|
||||
publish_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
-- lease
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TIMESTAMPTZ,
|
||||
-- retry
|
||||
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished
|
||||
ON attachment_jobs (next_attempt_at)
|
||||
WHERE published_at IS NULL AND status = 'queued';
|
||||
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
|
||||
ON attachment_jobs (lease_expires_at)
|
||||
WHERE status = 'processing';
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Индекс B для «стоимости записи»:
|
||||
-- покрывающий индекс, включающий text — растёт быстрее, помогает только
|
||||
-- покрывающим запросам. Накатывать отдельно, чтобы сравнить с индексом A.
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_covering
|
||||
ON messages (chat_id, created_at, id) INCLUDE (sender_id, text);
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Применяет миграции к БД, читая DATABASE_URL из файла секрета.
|
||||
# Устойчиво к повторному запуску (CREATE ... IF NOT EXISTS).
|
||||
URL_FILE="${DATABASE_URL_FILE:-/run/secrets/database_url}"
|
||||
DATABASE_URL="$(cat "$URL_FILE")"
|
||||
PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1 -q"
|
||||
|
||||
echo "waiting for postgres..."
|
||||
for i in $(seq 1 60); do
|
||||
if psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null 2>&1; then
|
||||
echo "postgres is ready"
|
||||
break
|
||||
fi
|
||||
echo " attempt $i: not ready yet"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null
|
||||
|
||||
for f in /app/migrations/*.sql; do
|
||||
echo "applying $f"
|
||||
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$f"
|
||||
done
|
||||
echo "migrations done"
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# ./scripts/restart.sh # 2 реплики (по умолчанию)
|
||||
# ./scripts/restart.sh 4 # 4 реплики
|
||||
REPLICAS="${1:-2}"
|
||||
|
||||
# Останов с сохранением volumes.
|
||||
docker compose down --remove-orphans
|
||||
|
||||
# Пересоздание сети + контейнеров с нужным числом реплик.
|
||||
API_REPLICAS="$REPLICAS" \
|
||||
docker compose --env-file .env -f compose.yaml up -d --build
|
||||
|
||||
# Ждём готовности с хоста.
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
|
||||
echo "ready after ${i}s (replicas=$REPLICAS)"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "stack did not become ready in 90s" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Все контейнеры сервиса api (включая Exited).
|
||||
# `docker compose ps -a -q api` возвращает их ID.
|
||||
STOPPED=""
|
||||
for id in $(docker compose ps -a -q api); do
|
||||
state=$(docker inspect --format '{{.State.Status}}' "$id")
|
||||
if [ "$state" = "exited" ] || [ "$state" = "created" ]; then
|
||||
STOPPED="$STOPPED $id"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$STOPPED" ]; then
|
||||
echo "all api containers already running"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for id in $STOPPED; do
|
||||
hn=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
docker start "$id" >/dev/null
|
||||
echo "started $id ($hn)"
|
||||
done
|
||||
|
||||
# Дождаться, пока LB увидит их снова (curl с хоста).
|
||||
i=0
|
||||
while [ "$i" -lt 30 ]; do
|
||||
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
|
||||
echo "ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "stack did not become ready in 30s" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
# Находим ID контейнера, чьё имя (HOSTNAME) совпадает с TARGET_INSTANCE_ID.
|
||||
CID=$(docker compose ps -q api | while read -r id; do
|
||||
name=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$name" = "$TARGET_INSTANCE_ID" ]; then
|
||||
echo "$id"
|
||||
break
|
||||
fi
|
||||
done)
|
||||
|
||||
if [ -z "$CID" ]; then
|
||||
echo "container with hostname $TARGET_INSTANCE_ID not found" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
docker stop "$CID"
|
||||
echo "stopped $CID ($TARGET_INSTANCE_ID)"
|
||||
@@ -0,0 +1,79 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"mtusi-messenger/src/domain"
|
||||
)
|
||||
|
||||
// bearerAuth извлекает токен, проверяет сессию и кладёт пользователя в контекст.
|
||||
// Basic на бизнес-эндпоинтах не принимается. X-User-Id игнорируется.
|
||||
func (h *Handler) bearerAuth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
header := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(header, "Bearer ") {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
|
||||
writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token")
|
||||
return
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(header, "Bearer "))
|
||||
if token == "" {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
|
||||
writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token")
|
||||
return
|
||||
}
|
||||
|
||||
u, err := h.Sessions.Lookup(r.Context(), token)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrUnauthorized):
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
|
||||
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid session")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.WithValue(r.Context(), userKey, *u)
|
||||
ctx = context.WithValue(ctx, tokenKey, token)
|
||||
next(w, r.WithContext(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
func userFrom(r *http.Request) (domain.User, bool) {
|
||||
u, ok := r.Context().Value(userKey).(domain.User)
|
||||
return u, ok
|
||||
}
|
||||
|
||||
func tokenFrom(r *http.Request) string {
|
||||
if v, ok := r.Context().Value(tokenKey).(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseBasicCredentials разбирает Authorization: Basic base64(username:password).
|
||||
func parseBasicCredentials(header string) (string, string, error) {
|
||||
if !strings.HasPrefix(header, "Basic ") {
|
||||
return "", "", errors.New("not basic")
|
||||
}
|
||||
raw := strings.TrimSpace(strings.TrimPrefix(header, "Basic "))
|
||||
decoded, err := base64.StdEncoding.DecodeString(raw)
|
||||
if err != nil {
|
||||
return "", "", errors.New("invalid base64")
|
||||
}
|
||||
parts := strings.SplitN(string(decoded), ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return "", "", errors.New("missing colon")
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"mtusi-messenger/src/domain"
|
||||
)
|
||||
|
||||
// writeError пишет ошибку в формате контракта.
|
||||
func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(domain.ErrorBody{
|
||||
Error: domain.ErrorDetail{
|
||||
Code: code,
|
||||
Message: message,
|
||||
RequestID: requestIDFrom(r),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// writeJSON пишет успешный JSON-ответ.
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"mtusi-messenger/src/domain"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
type Handler struct {
|
||||
Store *domain.Store
|
||||
Sessions *domain.SessionStore
|
||||
S3 *domain.S3Store
|
||||
MaxUpload int64
|
||||
}
|
||||
|
||||
// --- Health ---
|
||||
|
||||
func (h *Handler) Live(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// Ready проверяет БД (обязательная зависимость). Redis — кэш, его недоступность
|
||||
// не должна ронять readiness: API может работать, обходя кэш.
|
||||
func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) {
|
||||
if err := h.Store.Ping(r.Context()); err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// --- Users ---
|
||||
|
||||
type createUserReq struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) {
|
||||
var req createUserReq
|
||||
if err := decodeSingleJSON(r.Body, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
if !usernameRe.MatchString(req.Username) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid username")
|
||||
return
|
||||
}
|
||||
if n := len([]rune(req.DisplayName)); n < 1 || n > 100 || strings.TrimSpace(req.DisplayName) == "" {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid display_name")
|
||||
return
|
||||
}
|
||||
// Пароль: 12–128 code points, не состоит только из пробелов.
|
||||
if n := len([]rune(req.Password)); n < 12 || n > 128 || strings.TrimSpace(req.Password) == "" {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid password")
|
||||
return
|
||||
}
|
||||
u, err := h.Store.CreateUser(r.Context(), req.Username, req.DisplayName, req.Password)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrConflict):
|
||||
writeError(w, r, http.StatusConflict, "conflict", "username taken")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, u)
|
||||
}
|
||||
|
||||
func (h *Handler) CurrentUser(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
u, _ := userFrom(r)
|
||||
writeJSON(w, http.StatusOK, u)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
// --- Auth ---
|
||||
|
||||
func (h *Handler) CreateSession(w http.ResponseWriter, r *http.Request) {
|
||||
username, password, err := parseBasicCredentials(r.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
|
||||
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials")
|
||||
return
|
||||
}
|
||||
u, err := h.Store.VerifyPassword(r.Context(), username, password)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrUnauthorized):
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
|
||||
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
s, err := h.Sessions.Create(r.Context(), *u)
|
||||
if err != nil {
|
||||
if errors.Is(err, domain.ErrTimeout) {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, s)
|
||||
}
|
||||
|
||||
func (h *Handler) RevokeSession(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
token := tokenFrom(r)
|
||||
if err := h.Sessions.Revoke(r.Context(), token); err != nil {
|
||||
if errors.Is(err, domain.ErrTimeout) {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
// --- Chats ---
|
||||
|
||||
type createChatReq struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateChat(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
var req createChatReq
|
||||
if err := decodeSingleJSON(r.Body, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Title) == "" || len([]rune(req.Title)) > 100 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid title")
|
||||
return
|
||||
}
|
||||
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid member_ids")
|
||||
return
|
||||
}
|
||||
c, err := h.Store.CreateChat(r.Context(), req.Title, req.MemberIDs, me.ID)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid members")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, c)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) ListChats(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
limit := 50
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > 100 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit")
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
chats, next, err := h.Store.ListChats(r.Context(), me.ID, limit, cursor)
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, domain.ChatPage{Items: chats, NextCursor: next})
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) GetChat(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
c, err := h.Store.GetChat(r.Context(), chatID, me.ID)
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
|
||||
return
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, c)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
// --- Messages ---
|
||||
|
||||
type createMessageReq struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
|
||||
// Idempotency-Key обязателен.
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if key == "" {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required")
|
||||
return
|
||||
}
|
||||
if !idempotencyKeyRe.MatchString(key) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key")
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Читаем тело один раз и хешируем нормализованный JSON.
|
||||
rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024))
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body")
|
||||
return
|
||||
}
|
||||
var req createMessageReq
|
||||
if err := decodeSingleJSON(bytes.NewReader(rawBody), &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
if len([]rune(req.Text)) > 4000 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "message too long")
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message")
|
||||
return
|
||||
}
|
||||
bodyHash, err := domain.HashBody(rawBody)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
if len(req.Text) > 0 && len([]rune(req.Text)) == 0 {
|
||||
// всё пробелы, но вложения есть — допустимо
|
||||
}
|
||||
|
||||
attachmentIDs := req.AttachmentIDs
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = []string{}
|
||||
}
|
||||
msg, rec, replay, err := h.Store.AddMessageIdempotent(
|
||||
r.Context(), chatID, me.ID, req.Text, attachmentIDs, key, bodyHash)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
return
|
||||
case errors.Is(err, domain.ErrConflict):
|
||||
writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
if replay {
|
||||
// Возвращаем сохранённый 201 и то же Message.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(rec.StatusCode)
|
||||
_, _ = w.Write(rec.Response)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, msg)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) ListMessages(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
limit := 50
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > 100 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit")
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
items, next, err := h.Store.ListMessages(r.Context(), chatID, limit, cursor)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor")
|
||||
return
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "query timeout")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, domain.MessagePage{Items: items, NextCursor: next})
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func decodeSingleJSON(r io.Reader, v any) error {
|
||||
dec := json.NewDecoder(r)
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(v); err != nil {
|
||||
return err
|
||||
}
|
||||
if dec.More() {
|
||||
return errors.New("trailing data after JSON value")
|
||||
}
|
||||
// Пробуем прочитать ещё один токен — если это не EOF, значит есть мусор.
|
||||
var extra any
|
||||
if err := dec.Decode(&extra); err != io.EOF {
|
||||
return errors.New("trailing data after JSON value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Attachments ---
|
||||
|
||||
// safeFilename отбрасывает пути и опасные имена; используется только для отображения.
|
||||
func safeFilename(name string) (string, bool) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || len(name) > 255 {
|
||||
return "", false
|
||||
}
|
||||
if strings.ContainsAny(name, "/\\") || strings.Contains(name, "..") {
|
||||
return "", false
|
||||
}
|
||||
for _, r := range name {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
func errorCodeFor(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
return "payload_too_large"
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
return "unsupported_media_type"
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
return "invalid_image"
|
||||
default:
|
||||
return "storage_error"
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
|
||||
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
filename, ok := safeFilename(r.URL.Query().Get("filename"))
|
||||
if !ok {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid filename")
|
||||
return
|
||||
}
|
||||
|
||||
stagingKey := domain.StagingPrefix + chatID + "/" + uuid.NewString()
|
||||
pendingID, err := h.Store.CreateAttachmentPending(
|
||||
r.Context(), chatID, me.ID, stagingKey, filename)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid attachment metadata")
|
||||
case errors.Is(err, domain.ErrTimeout):
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
res, err := h.S3.PutValidatedImage(r.Context(), stagingKey, r.Body)
|
||||
if err != nil {
|
||||
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, errorCodeFor(err))
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "upload_failed")
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large")
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
writeError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "only PNG/JPEG")
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid image")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Атомарно: attachment queued + outbox-строка. 202 только после COMMIT.
|
||||
att, err := h.Store.EnqueueAttachment(
|
||||
r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256,
|
||||
res.Width, res.Height, domain.PipelineVersion)
|
||||
if err != nil {
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "enqueue_failed")
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, att)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
id := r.PathValue("attachment_id")
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
|
||||
switch {
|
||||
case err == nil:
|
||||
writeJSON(w, http.StatusOK, att)
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
id := r.PathValue("attachment_id")
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
|
||||
if err != nil {
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
switch att.Status {
|
||||
case domain.AttachmentReady:
|
||||
// ok
|
||||
case domain.AttachmentFailed:
|
||||
writeError(w, r, http.StatusConflict, "conflict", "attachment failed")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusConflict, "conflict", "attachment not ready")
|
||||
return
|
||||
}
|
||||
|
||||
variant := r.URL.Query().Get("variant")
|
||||
if variant == "" {
|
||||
variant = "original"
|
||||
}
|
||||
var key string
|
||||
if variant == "original" {
|
||||
key, err = h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
|
||||
} else {
|
||||
key, _, err = h.Store.GetVariantKey(r.Context(), att.ID, att.PipelineVersion, variant)
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
|
||||
rc, contentType, size, err := h.S3.GetObject(r.Context(), key)
|
||||
if err != nil {
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "file missing in storage")
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
w.Header().Set("Content-Disposition", `inline; filename="`+strings.ReplaceAll(att.Filename, `"`, "")+`"`)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
written, err := io.Copy(w, rc)
|
||||
if err != nil || written != size {
|
||||
// Соединение уже частично отправлено, но клиент получит обрезанный файл.
|
||||
// Логируем это для отладки.
|
||||
log.Printf("download truncated: written=%d expected=%d err=%v", written, size, err)
|
||||
}
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) ReplayAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Только owner или участник чата; упрощённо — участник чата.
|
||||
me, _ := userFrom(r)
|
||||
id := r.PathValue("attachment_id")
|
||||
if _, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID); err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
err := h.Store.ReplayFailed(r.Context(), id)
|
||||
if errors.Is(err, domain.ErrConflict) {
|
||||
writeError(w, r, http.StatusConflict, "conflict", "not in failed state")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
})(w, r)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type ctxKey string
|
||||
|
||||
const (
|
||||
requestIDKey ctxKey = "request_id"
|
||||
userKey ctxKey = "user"
|
||||
tokenKey ctxKey = "token"
|
||||
)
|
||||
|
||||
func requestIDFrom(r *http.Request) string {
|
||||
if v, ok := r.Context().Value(requestIDKey).(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func withRequestID(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.Header.Get("X-Request-Id")
|
||||
if id == "" {
|
||||
id = uuid.NewString()
|
||||
}
|
||||
w.Header().Set("X-Request-Id", id)
|
||||
ctx := context.WithValue(r.Context(), requestIDKey, id)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health).
|
||||
// ID генерируется один раз при старте процесса — стабилен в пределах жизни.
|
||||
func withInstanceID(instanceID string) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package api
|
||||
|
||||
import "net/http"
|
||||
|
||||
func NewRouter(h *Handler, instanceID string) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /health/live", h.Live)
|
||||
mux.HandleFunc("GET /health/ready", h.Ready)
|
||||
mux.HandleFunc("POST /api/v1/users", h.CreateUser)
|
||||
mux.HandleFunc("GET /api/v1/users/me", h.CurrentUser)
|
||||
mux.HandleFunc("POST /api/v1/auth/sessions", h.CreateSession)
|
||||
mux.HandleFunc("DELETE /api/v1/auth/sessions/current", h.RevokeSession)
|
||||
mux.HandleFunc("POST /api/v1/chats", h.CreateChat)
|
||||
mux.HandleFunc("GET /api/v1/chats", h.ListChats)
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat)
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage)
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages)
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment)
|
||||
mux.HandleFunc("POST /api/v1/attachments/{attachment_id}/replay", h.ReplayAttachment)
|
||||
|
||||
// Порядок: instanceID снаружи, requestID внутри.
|
||||
return withInstanceID(instanceID)(withRequestID(mux))
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
Env string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
SessionStore string
|
||||
RedisRequired bool
|
||||
InstanceID string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKeyID string
|
||||
S3SecretKey string
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
S3GCGrace time.Duration
|
||||
}
|
||||
|
||||
func getenv(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func readSecretFromFile(path string) string {
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
port := 8080
|
||||
if v := os.Getenv("APP_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil {
|
||||
port = p
|
||||
}
|
||||
}
|
||||
ttl := 3600
|
||||
if v := os.Getenv("SESSION_TTL_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
ttl = n
|
||||
}
|
||||
}
|
||||
|
||||
// DATABASE_URL и CACHE_URL могут прийти либо напрямую, либо через *_FILE.
|
||||
dbURL := os.Getenv("DATABASE_URL")
|
||||
if dbURL == "" {
|
||||
dbURL = readSecretFromFile(os.Getenv("DATABASE_URL_FILE"))
|
||||
}
|
||||
cacheURL := os.Getenv("CACHE_URL")
|
||||
if cacheURL == "" {
|
||||
cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE"))
|
||||
}
|
||||
|
||||
instanceID := os.Getenv("INSTANCE_ID")
|
||||
if instanceID == "" {
|
||||
instanceID = os.Getenv("HOSTNAME")
|
||||
}
|
||||
if instanceID == "" {
|
||||
instanceID = "api-" + randomShortID()
|
||||
}
|
||||
|
||||
store := getenv("SESSION_STORE", "postgres")
|
||||
|
||||
s3Access := os.Getenv("S3_ACCESS_KEY_ID")
|
||||
if s3Access == "" {
|
||||
s3Access = readSecretFromFile(os.Getenv("S3_ACCESS_KEY_ID_FILE"))
|
||||
}
|
||||
s3Secret := os.Getenv("S3_SECRET_ACCESS_KEY")
|
||||
if s3Secret == "" {
|
||||
s3Secret = readSecretFromFile(os.Getenv("S3_SECRET_ACCESS_KEY_FILE"))
|
||||
}
|
||||
maxUpload := int64(10485760)
|
||||
if v := os.Getenv("MAX_UPLOAD_BYTES"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
maxUpload = n
|
||||
}
|
||||
}
|
||||
maxPixels := int64(40000000)
|
||||
if v := os.Getenv("MAX_IMAGE_PIXELS"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
maxPixels = n
|
||||
}
|
||||
}
|
||||
gcGrace := 600 * time.Second
|
||||
if v := os.Getenv("S3_GC_GRACE_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
gcGrace = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
return Config{
|
||||
Host: getenv("APP_HOST", "0.0.0.0"),
|
||||
Port: port,
|
||||
Env: getenv("APP_ENV", "lab"),
|
||||
DatabaseURL: dbURL,
|
||||
CacheURL: cacheURL,
|
||||
SessionTTL: time.Duration(ttl) * time.Second,
|
||||
SessionStore: store,
|
||||
RedisRequired: store == "redis",
|
||||
InstanceID: instanceID,
|
||||
S3Endpoint: getenv("S3_ENDPOINT", ""),
|
||||
S3Region: getenv("S3_REGION", "us-east-1"),
|
||||
S3Bucket: getenv("S3_BUCKET", "messenger-attachments"),
|
||||
S3AccessKeyID: s3Access,
|
||||
S3SecretKey: s3Secret,
|
||||
MaxUploadBytes: maxUpload,
|
||||
MaxImagePixels: maxPixels,
|
||||
S3GCGrace: gcGrace,
|
||||
}
|
||||
}
|
||||
|
||||
func randomShortID() string {
|
||||
b := make([]byte, 6)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "unknown"
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package domain
|
||||
|
||||
const (
|
||||
AttachmentQueued = "queued"
|
||||
AttachmentProcessing = "processing"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentFailed = "failed"
|
||||
)
|
||||
|
||||
const PipelineVersion = 1
|
||||
|
||||
var RequiredVariants = []string{"thumbnail", "crop", "optimized"}
|
||||
@@ -0,0 +1,613 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// CreateAttachmentPending резервирует строку ДО записи в S3.
|
||||
// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится.
|
||||
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, stagingKey, filename string) (string, error) {
|
||||
id := uuid.NewString()
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO attachments (id, chat_id, owner_id, object_key, staging_key, filename,
|
||||
content_type, size_bytes, sha256, width, height,
|
||||
pipeline_version, status)
|
||||
VALUES ($1,$2,$3,$4,$4,$5,'application/octet-stream',0,repeat('0',64),0,0,1,'queued')`,
|
||||
id, chatID, ownerID, stagingKey, filename)
|
||||
if err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// MarkAttachmentFailed помечает pending как failed (компенсация).
|
||||
func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachments SET status='failed', error_code=$2, updated_at=now()
|
||||
WHERE id=$1 AND status='queued'`, id, code)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// ScheduleObjectGC ставит object_key в очередь на удаление (grace period).
|
||||
func (s *Store) ScheduleObjectGC(ctx context.Context, objectKey, reason string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO attachment_gc (object_key, reason) VALUES ($1,$2)
|
||||
ON CONFLICT (object_key) DO NOTHING`, objectKey, reason)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// GetAttachmentForUser возвращает вложение, только если пользователь — участник чата.
|
||||
func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*Attachment, error) {
|
||||
var a Attachment
|
||||
var errCode *string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.ErrorCode = errCode
|
||||
a.Variants = []Variant{}
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT variant, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2
|
||||
AND variant = ANY($3)
|
||||
ORDER BY variant`,
|
||||
a.ID, a.PipelineVersion, RequiredVariants)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v Variant
|
||||
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes,
|
||||
&v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.Variants = append(a.Variants, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// GetAttachmentObjectKey — только для внутреннего download после проверки ACL.
|
||||
func (s *Store) GetAttachmentObjectKey(ctx context.Context, id string) (string, error) {
|
||||
var key string
|
||||
err := s.pool.QueryRow(ctx, `SELECT object_key FROM attachments WHERE id=$1`, id).Scan(&key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// ValidateAttachmentIDsForChat проверяет, что все вложения принадлежат чату,
|
||||
// имеют статус ready и лимит 10. Возвращает ErrInvalid/ErrNotFound.
|
||||
func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string, ids []string) error {
|
||||
if len(ids) > 10 {
|
||||
return ErrInvalid
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range ids {
|
||||
if seen[id] {
|
||||
return ErrInvalid
|
||||
}
|
||||
seen[id] = true
|
||||
var status string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`,
|
||||
id, chatID).Scan(&status)
|
||||
if err == nil && status == AttachmentFailed {
|
||||
return ErrInvalid
|
||||
}
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReconcileOrphans: удаляет объекты из attachment_gc старше grace, которых нет в БД,
|
||||
// и «призрачные» объекты pending старше grace. Вызывается из фонового тикера.
|
||||
func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Duration) error {
|
||||
// 1) GC-очередь: удалить объекты в S3, затем строку.
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT object_key FROM attachment_gc WHERE created_at < now() - $1::interval LIMIT 100`,
|
||||
grace.String())
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
rows.Close()
|
||||
return classifyDBError(err)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
rows.Close()
|
||||
for _, k := range keys {
|
||||
if err := s3.DeleteObject(ctx, k); err != nil {
|
||||
continue
|
||||
}
|
||||
_, _ = s.pool.Exec(ctx, `DELETE FROM attachment_gc WHERE object_key=$1`, k)
|
||||
}
|
||||
|
||||
// 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет.
|
||||
prow, err := s.pool.Query(ctx,
|
||||
`SELECT a.id, a.staging_key FROM attachments a
|
||||
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id
|
||||
WHERE a.status='queued' AND j.job_id IS NULL
|
||||
AND a.created_at < now() - $1::interval LIMIT 100`,
|
||||
grace.String())
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
type pend struct{ id, key string }
|
||||
var pendings []pend
|
||||
for prow.Next() {
|
||||
var p pend
|
||||
if err := prow.Scan(&p.id, &p.key); err != nil {
|
||||
prow.Close()
|
||||
return classifyDBError(err)
|
||||
}
|
||||
pendings = append(pendings, p)
|
||||
}
|
||||
prow.Close()
|
||||
for _, p := range pendings {
|
||||
_ = s3.DeleteObject(ctx, p.key)
|
||||
_, _ = s.pool.Exec(ctx, `DELETE FROM attachments WHERE id=$1 AND status='pending'`, p.id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnqueueAttachment: атомарно фиксируем метаданные приёма и кладём outbox-строку.
|
||||
// 202 отвечаем только после успешного COMMIT этой транзакции.
|
||||
func (s *Store) EnqueueAttachment(
|
||||
ctx context.Context, attachmentID, contentType string,
|
||||
size int64, sha string, w, h int, pipelineVersion int,
|
||||
) (*Attachment, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var a Attachment
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachments
|
||||
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
|
||||
pipeline_version=$7, status='queued', updated_at=now()
|
||||
WHERE id=$1 AND status='queued'
|
||||
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at, pipeline_version`,
|
||||
attachmentID, contentType, size, sha, w, h, pipelineVersion,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes,
|
||||
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, pipeline_version, status, next_attempt_at)
|
||||
VALUES ($1,$2,$3,'queued', now())`,
|
||||
uuid.NewString(), attachmentID, pipelineVersion,
|
||||
); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.Variants = []Variant{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// ClaimJob — берём одну доступную задачу под lease.
|
||||
// Работает и для queued, и для протухшего processing (reaper тем же запросом).
|
||||
func (s *Store) ClaimJob(ctx context.Context, workerID string, lease time.Duration) (*Job, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var j Job
|
||||
err = tx.QueryRow(ctx, `
|
||||
WITH picked AS (
|
||||
SELECT job_id FROM attachment_jobs
|
||||
WHERE status IN ('queued','processing')
|
||||
AND next_attempt_at <= now()
|
||||
AND (status='queued' OR lease_expires_at < now())
|
||||
ORDER BY next_attempt_at
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE attachment_jobs aj
|
||||
SET status='processing',
|
||||
lease_owner=$1,
|
||||
lease_expires_at = now() + $2::interval,
|
||||
updated_at = now()
|
||||
FROM picked
|
||||
WHERE aj.job_id = picked.job_id
|
||||
RETURNING aj.job_id, aj.attachment_id, aj.pipeline_version,
|
||||
aj.attempt, aj.status, aj.next_attempt_at, aj.last_error_code`,
|
||||
workerID, lease.String(),
|
||||
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.Status,
|
||||
&j.NextAttemptAt, &j.LastError)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// MarkReady: атомарно пишем все варианты (идемпотентно по ключу) и завершаем job.
|
||||
// Конкурентная попытка не перезапишет существующий вариант — ON CONFLICT DO NOTHING.
|
||||
func (s *Store) MarkReady(
|
||||
ctx context.Context, jobID, attachmentID string, pipelineVersion int,
|
||||
finalOriginalKey string, variants []Variant,
|
||||
) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// Если job уже done — no-op (redelivery после ack).
|
||||
var status string
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT status FROM attachment_jobs WHERE job_id=$1 FOR UPDATE`, jobID,
|
||||
).Scan(&status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if status == "done" {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if status == "failed" {
|
||||
return ErrConflict // нельзя откатывать failed→ready; см. replay
|
||||
}
|
||||
|
||||
// Записать оригинал в final prefix.
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments
|
||||
SET object_key=$2, status='ready', ready_at=now(), updated_at=now()
|
||||
WHERE id=$1 AND pipeline_version=$3`,
|
||||
attachmentID, finalOriginalKey, pipelineVersion,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
for _, v := range variants {
|
||||
if v.Name == "original" {
|
||||
continue // original живёт в attachments.object_key, не в variants
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_variants
|
||||
(attachment_id, pipeline_version, variant, object_key,
|
||||
content_type, size_bytes, width, height, sha256)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`,
|
||||
attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name,
|
||||
v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachment_jobs
|
||||
SET status='done', lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// RetryJob: временная ошибка — вернуть в очередь с backoff+jitter.
|
||||
// exhausted=true — если попытки кончились.
|
||||
func (s *Store) RetryJob(
|
||||
ctx context.Context, jobID string, errorCode string,
|
||||
maxAttempts int, baseBackoff time.Duration,
|
||||
) (exhausted bool, err error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var attempt int
|
||||
var attachmentID string
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT attempt, attachment_id FROM attachment_jobs
|
||||
WHERE job_id=$1 FOR UPDATE`, jobID,
|
||||
).Scan(&attempt, &attachmentID)
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
|
||||
if attempt >= maxAttempts {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='failed', last_error_code=$2,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, errorCode); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status='failed', failed_at=now(),
|
||||
error_code=$2, updated_at=now()
|
||||
WHERE id=$1`, attachmentID, errorCode); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
log.Printf("retry job=%s exhausted attempts=%d code=%s",
|
||||
jobID, attempt, errorCode)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// backoff: base * 2^(attempt-1) + jitter
|
||||
backoff := baseBackoff * time.Duration(1<<(attempt-1))
|
||||
jitter := time.Duration(rand.Int63n(int64(backoff / 4)))
|
||||
next := time.Now().Add(backoff + jitter)
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET attempt = attempt + 1,
|
||||
status = 'queued',
|
||||
last_error_code=$2,
|
||||
next_attempt_at=$3,
|
||||
published_at=NULL,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, errorCode, next); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
// Логируем зафиксированный retry — используется в student-tests/backoff.py
|
||||
log.Printf("retry job=%s attempt=%d next_attempt=%s backoff=%s jitter=%s code=%s",
|
||||
jobID, attempt+1, next.UTC().Format(time.RFC3339Nano),
|
||||
backoff, jitter, errorCode)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox.
|
||||
func (s *Store) RequeueStuck(ctx context.Context) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
|
||||
next_attempt_at = now(), published_at = NULL, updated_at = now()
|
||||
WHERE status='processing' AND lease_expires_at < now()`)
|
||||
if err != nil {
|
||||
return 0, classifyDBError(err)
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// ReplayFailed — операторский replay (уровень 4, CLI/endpoint).
|
||||
func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='queued', attempt=1, next_attempt_at=now(),
|
||||
published_at=NULL,
|
||||
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments SET status='queued', error_code=NULL, failed_at=NULL,
|
||||
updated_at=now()
|
||||
WHERE id=$1 AND status='failed'`, attachmentID); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// ClaimJobByID — берёт конкретную job под lease. Нужен для redelivery:
|
||||
// если worker упал после publish, повторно доставленное сообщение должно
|
||||
// либо взять ту же работу (lease протух), либо быть no-op (status=done).
|
||||
func (s *Store) ClaimJobByID(ctx context.Context, jobID, workerID string, lease time.Duration) (*Job, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var j Job
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='processing',
|
||||
lease_owner=$2,
|
||||
lease_expires_at = now() + $3::interval,
|
||||
updated_at = now()
|
||||
WHERE job_id=$1
|
||||
AND status IN ('queued','processing')
|
||||
AND (status='queued' OR lease_expires_at < now())
|
||||
RETURNING job_id, attachment_id, pipeline_version, attempt,
|
||||
status, next_attempt_at, last_error_code`,
|
||||
jobID, workerID, lease.String(),
|
||||
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt,
|
||||
&j.Status, &j.NextAttemptAt, &j.LastError)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// GetStagingKey — доверенные метаданные для worker’а (не из сообщения брокера).
|
||||
func (s *Store) GetStagingKey(ctx context.Context, attachmentID string) (string, string, error) {
|
||||
var key, ct string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT staging_key, content_type FROM attachments WHERE id=$1`, attachmentID,
|
||||
).Scan(&key, &ct)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", classifyDBError(err)
|
||||
}
|
||||
return key, ct, nil
|
||||
}
|
||||
|
||||
// VariantsExist — no-op-проверка на повторную доставку уже готовой работы.
|
||||
func (s *Store) VariantsExist(ctx context.Context, attachmentID string, pv int) (bool, error) {
|
||||
var n int
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT count(*) FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2`,
|
||||
attachmentID, pv,
|
||||
).Scan(&n)
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
return n >= len(RequiredVariants), nil
|
||||
}
|
||||
|
||||
// MarkFailedPermanent — терминальный failed без retry (постоянная ошибка).
|
||||
func (s *Store) MarkFailedPermanent(ctx context.Context, jobID, attachmentID, code string) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET status='failed', last_error_code=$2,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1 AND status='processing'`, jobID, code); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments SET status='failed', error_code=$2,
|
||||
failed_at=now(), updated_at=now()
|
||||
WHERE id=$1`, attachmentID, code); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
type UnpublishedJob struct {
|
||||
JobID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
}
|
||||
|
||||
// SelectUnpublishedJobs — выборка outbox’ом. Транзакция не нужна:
|
||||
// published_at ставим отдельным UPDATE после XADD; повторная публикация
|
||||
// допустима (worker идемпотентен).
|
||||
func (s *Store) SelectUnpublishedJobs(ctx context.Context, limit int) ([]UnpublishedJob, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT job_id, attachment_id, pipeline_version, attempt
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL AND status='queued'
|
||||
AND next_attempt_at <= now()
|
||||
ORDER BY next_attempt_at
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []UnpublishedJob
|
||||
for rows.Next() {
|
||||
var u UnpublishedJob
|
||||
if err := rows.Scan(&u.JobID, &u.AttachmentID, &u.PipelineVersion, &u.Attempt); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
out = append(out, u)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MarkPublished — фиксируем факт XADD (publisher confirm).
|
||||
func (s *Store) MarkPublished(ctx context.Context, jobID string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET published_at=now(),
|
||||
publish_attempts=publish_attempts+1, updated_at=now()
|
||||
WHERE job_id=$1`, jobID)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// BumpPublishAttempt — XADD упал; фиксируем причину, повторим в следующем цикле.
|
||||
func (s *Store) BumpPublishAttempt(ctx context.Context, jobID, code string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET publish_attempts=publish_attempts+1,
|
||||
last_error_code=$2, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, code)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
func (s *Store) GetVariantKey(ctx context.Context, attachmentID string, pv int, variant string) (string, string, error) {
|
||||
var key, ct string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT object_key, content_type FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 AND variant=$3`,
|
||||
attachmentID, pv, variant).Scan(&key, &ct)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", classifyDBError(err)
|
||||
}
|
||||
return key, ct, nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
StreamName = "attachment.jobs"
|
||||
ConsumerGroup = "workers"
|
||||
)
|
||||
|
||||
type JobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
}
|
||||
|
||||
type Broker struct{ rdb *redis.Client }
|
||||
|
||||
func NewBroker(ctx context.Context, url string) (*Broker, error) {
|
||||
opt, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rdb := redis.NewClient(opt)
|
||||
if err := rdb.Ping(ctx).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Группа создаётся идемпотентно.
|
||||
err = rdb.XGroupCreateMkStream(ctx, StreamName, ConsumerGroup, "0").Err()
|
||||
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
||||
return nil, err
|
||||
}
|
||||
return &Broker{rdb: rdb}, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Publish(ctx context.Context, m JobMessage) (string, error) {
|
||||
buf, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return b.rdb.XAdd(ctx, &redis.XAddArgs{
|
||||
Stream: StreamName,
|
||||
Values: map[string]any{"payload": buf},
|
||||
}).Result()
|
||||
}
|
||||
|
||||
// Consume — блокирующее чтение из consumer group; ack делает вызывающий.
|
||||
func (b *Broker) Consume(ctx context.Context, consumer string) ([]redis.XMessage, error) {
|
||||
res, err := b.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: ConsumerGroup, Consumer: consumer,
|
||||
Streams: []string{StreamName, ">"},
|
||||
Count: 1, Block: 5 * time.Second,
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return res[0].Messages, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Ack(ctx context.Context, id string) error {
|
||||
return b.rdb.XAck(ctx, StreamName, ConsumerGroup, id).Err()
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// SessionCache — необязательный кэш сессий поверх Redis.
|
||||
// Источник истины — PostgreSQL. Кэш ускоряет проверку токена.
|
||||
type SessionCache struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func NewSessionCache(ctx context.Context, url string) (*SessionCache, error) {
|
||||
opt, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Жёсткие короткие таймауты: Redis — кэш, а не критический путь.
|
||||
opt.DialTimeout = 1 * time.Second
|
||||
opt.ReadTimeout = 500 * time.Millisecond
|
||||
opt.WriteTimeout = 500 * time.Millisecond
|
||||
opt.MaxRetries = 0 // без автоматических ретраев: одна попытка — и уходим в БД
|
||||
opt.PoolTimeout = 1 * time.Second
|
||||
|
||||
rdb := redis.NewClient(opt)
|
||||
|
||||
// Проверка связи при старте — короткая.
|
||||
c, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
if err := rdb.Ping(c).Err(); err != nil {
|
||||
_ = rdb.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &SessionCache{rdb: rdb}, nil
|
||||
}
|
||||
|
||||
func (c *SessionCache) userKey(userID string) string { return "user:" + userID }
|
||||
|
||||
// opCtx возвращает контекст с коротким таймаутом для каждой операции.
|
||||
// Не наследуем HTTP-контекст напрямую, чтобы не блокировать ответ клиенту.
|
||||
func (c *SessionCache) opCtx() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
}
|
||||
|
||||
// SetUser кладёт публичные данные пользователя в кэш с TTL до конца сессии.
|
||||
func (c *SessionCache) SetUser(ctx context.Context, userID string, u User, expires time.Time) {
|
||||
ttl := time.Until(expires)
|
||||
if ttl <= 0 {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(u)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
opCtx, cancel := c.opCtx()
|
||||
defer cancel()
|
||||
_ = c.rdb.Set(opCtx, c.userKey(userID), b, ttl).Err()
|
||||
}
|
||||
|
||||
func (c *SessionCache) GetUser(ctx context.Context, userID string) (User, bool) {
|
||||
opCtx, cancel := c.opCtx()
|
||||
defer cancel()
|
||||
b, err := c.rdb.Get(opCtx, c.userKey(userID)).Bytes()
|
||||
if err != nil {
|
||||
return User{}, false
|
||||
}
|
||||
var u User
|
||||
if err := json.Unmarshal(b, &u); err != nil {
|
||||
return User{}, false
|
||||
}
|
||||
return u, true
|
||||
}
|
||||
|
||||
// DelUser — на случай удаления пользователя (в lab2 не вызывается).
|
||||
func (c *SessionCache) DelUser(ctx context.Context, userID string) error {
|
||||
opCtx, cancel := c.opCtx()
|
||||
defer cancel()
|
||||
return c.rdb.Del(opCtx, c.userKey(userID)).Err()
|
||||
}
|
||||
|
||||
func (c *SessionCache) Ping(ctx context.Context) error {
|
||||
c2, cancel := context.WithTimeout(ctx, 1*time.Second)
|
||||
defer cancel()
|
||||
return c.rdb.Ping(c2).Err()
|
||||
}
|
||||
|
||||
func (c *SessionCache) Close() error { return c.rdb.Close() }
|
||||
@@ -0,0 +1,75 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// IdempotencyRecord — сохранённый результат POST.
|
||||
type IdempotencyRecord struct {
|
||||
StatusCode int
|
||||
Response json.RawMessage
|
||||
}
|
||||
|
||||
// LookupIdempotency возвращает запись, если ключ уже использован.
|
||||
// Второй аргумент — совпадает ли хеш тела с сохранённым.
|
||||
func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) {
|
||||
var (
|
||||
storedHash string
|
||||
rec IdempotencyRecord
|
||||
)
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`,
|
||||
userID, method, path, key,
|
||||
).Scan(&storedHash, &rec.StatusCode, &rec.Response)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, classifyDBError(err)
|
||||
}
|
||||
if storedHash != bodyHash {
|
||||
return nil, true, ErrConflict
|
||||
}
|
||||
return &rec, true, nil
|
||||
}
|
||||
|
||||
// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный
|
||||
// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency.
|
||||
func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
|
||||
userID, method, path, key, bodyHash, status, resp)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// HashBody — стабильный хеш JSON независимо от порядка ключей.
|
||||
func HashBody(raw []byte) (string, error) {
|
||||
var v any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return "", err
|
||||
}
|
||||
normalized, err := json.Marshal(v) // map keys сортируются
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(normalized)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// CleanupIdempotency — можно звать из фонового тикера (опционально).
|
||||
func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`,
|
||||
olderThan.String())
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
type Job struct {
|
||||
JobID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
Status string
|
||||
NextAttemptAt time.Time
|
||||
LastError *string
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type MessagePage struct {
|
||||
Items []Message `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type ChatPage struct {
|
||||
Items []Chat `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
User User `json:"user"`
|
||||
}
|
||||
|
||||
type ErrorBody struct {
|
||||
Error ErrorDetail `json:"error"`
|
||||
}
|
||||
|
||||
type ErrorDetail struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ObjectKey string `json:"-"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
PipelineVersion int `json:"-"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrInvalid = errors.New("invalid request")
|
||||
ErrUnauthorized = errors.New("unauthorized")
|
||||
ErrTimeout = errors.New("timeout")
|
||||
ErrUnavailable = errors.New("unavailable")
|
||||
ErrTooLarge = errors.New("payload too large")
|
||||
ErrUnsupported = errors.New("unsupported media type")
|
||||
ErrNotImage = errors.New("not a valid image")
|
||||
)
|
||||
@@ -0,0 +1,619 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewStore(ctx context.Context, dsn string) (*Store, error) {
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
cfg.MaxConns = 10
|
||||
cfg.MinConns = 1
|
||||
cfg.MaxConnLifetime = time.Hour
|
||||
cfg.MaxConnIdleTime = 5 * time.Minute
|
||||
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
|
||||
|
||||
cfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
|
||||
_, err := conn.Exec(ctx, `
|
||||
SET statement_timeout = '5s';
|
||||
SET lock_timeout = '2s';
|
||||
SET idle_in_transaction_session_timeout = '5s';
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
ctxPing, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := pool.Ping(ctxPing); err != nil {
|
||||
pool.Close()
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &Store{pool: pool}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() { s.pool.Close() }
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
c, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
return s.pool.Ping(c)
|
||||
}
|
||||
|
||||
// Argon2id параметры — OWASP baseline: m=19456 KiB (19 MiB), t=2, p=1.
|
||||
// Хеш кодируется в формате PHC: $argon2id$v=19$m=19456,t=2,p=1$<salt>$<hash>
|
||||
const (
|
||||
argonMemory uint32 = 19456
|
||||
argonTime uint32 = 2
|
||||
argonThreads uint8 = 1
|
||||
argonKeyLen uint32 = 32
|
||||
argonSaltLen = 16
|
||||
)
|
||||
|
||||
func hashPassword(password string) (string, error) {
|
||||
salt := make([]byte, argonSaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
|
||||
b64Salt := base64.RawStdEncoding.EncodeToString(salt)
|
||||
b64Key := base64.RawStdEncoding.EncodeToString(key)
|
||||
// PHC string; параметры можно будет менять в будущем, старые хеши останутся валидными.
|
||||
return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$%s$%s",
|
||||
argonMemory, argonTime, argonThreads, b64Salt, b64Key), nil
|
||||
}
|
||||
|
||||
func verifyPassword(password, encoded string) bool {
|
||||
// Минимальный парсер PHC-строки.
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[1] != "argon2id" {
|
||||
return false
|
||||
}
|
||||
var mem, time uint32
|
||||
var threads uint8
|
||||
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &time, &threads); err != nil {
|
||||
return false
|
||||
}
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
wantKey, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
gotKey := argon2.IDKey([]byte(password), salt, time, mem, threads, uint32(len(wantKey)))
|
||||
return subtle.ConstantTimeCompare(gotKey, wantKey) == 1
|
||||
}
|
||||
|
||||
// --- Users ---
|
||||
|
||||
func (s *Store) CreateUser(ctx context.Context, username, displayName, password string) (*User, error) {
|
||||
hash, err := hashPassword(password)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
id := uuid.NewString()
|
||||
var u User
|
||||
err = s.pool.QueryRow(ctx,
|
||||
`INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ($1,$2,$3,$4)
|
||||
RETURNING id, username, display_name, created_at`,
|
||||
id, username, displayName, string(hash),
|
||||
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrConflict
|
||||
}
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetUserByID(ctx context.Context, id string) (*User, error) {
|
||||
var u User
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT id, username, display_name, created_at FROM users WHERE id=$1`, id,
|
||||
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// VerifyPassword возвращает пользователя, если пароль верный; иначе ErrUnauthorized.
|
||||
func (s *Store) VerifyPassword(ctx context.Context, username, password string) (*User, error) {
|
||||
var u User
|
||||
var hash string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username=$1`,
|
||||
username,
|
||||
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt, &hash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// Тратим сопоставимое время на проверку фиктивного хеша,
|
||||
// чтобы не различать "нет пользователя" и "неверный пароль" по таймингу.
|
||||
_ = verifyPassword(password, "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
|
||||
return nil, ErrUnauthorized
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
if !verifyPassword(password, hash) {
|
||||
return nil, ErrUnauthorized
|
||||
}
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
// --- Chats ---
|
||||
|
||||
func (s *Store) CreateChat(ctx context.Context, title string, memberIDs []string, creatorID string) (*Chat, error) {
|
||||
// Проверка входных данных.
|
||||
seen := map[string]bool{}
|
||||
for _, id := range memberIDs {
|
||||
if seen[id] {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
if !seen[creatorID] {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// Проверяем, что все участники существуют.
|
||||
for id := range seen {
|
||||
var exists bool
|
||||
if err := tx.QueryRow(ctx, `SELECT true FROM users WHERE id=$1`, id).Scan(&exists); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
}
|
||||
|
||||
chatID := uuid.NewString()
|
||||
var c Chat
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO chats (id, title) VALUES ($1,$2) RETURNING id, title, created_at`,
|
||||
chatID, title,
|
||||
).Scan(&c.ID, &c.Title, &c.CreatedAt)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
for id := range seen {
|
||||
if _, err := tx.Exec(ctx,
|
||||
`INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2)`,
|
||||
c.ID, id,
|
||||
); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
// Порядок member_ids — как в запросе.
|
||||
c.MemberIDs = append([]string(nil), memberIDs...)
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetChat(ctx context.Context, chatID, userID string) (*Chat, error) {
|
||||
var exists bool
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
|
||||
chatID, userID,
|
||||
).Scan(&exists)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
var c Chat
|
||||
err = s.pool.QueryRow(ctx,
|
||||
`SELECT id, title, created_at FROM chats WHERE id=$1`, chatID,
|
||||
).Scan(&c.ID, &c.Title, &c.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chatID)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var uid string
|
||||
if err := rows.Scan(&uid); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
c.MemberIDs = append(c.MemberIDs, uid)
|
||||
}
|
||||
return &c, rows.Err()
|
||||
}
|
||||
|
||||
// ListChats возвращает чаты пользователя с keyset-пагинацией по (created_at, id).
|
||||
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cursor string) ([]Chat, *string, error) {
|
||||
var rows pgx.Rows
|
||||
var err error
|
||||
|
||||
if cursor == "" {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = $1
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $2`, userID, limit+1)
|
||||
} else {
|
||||
var cCreated time.Time
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT created_at FROM chats WHERE id=$1`, cursor,
|
||||
).Scan(&cCreated); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, ErrInvalid
|
||||
}
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = $1
|
||||
AND (c.created_at, c.id) > ($2, $3)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4`, userID, cCreated, cursor, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var chats []Chat
|
||||
for rows.Next() {
|
||||
var c Chat
|
||||
if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
chats = append(chats, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
var next *string
|
||||
if len(chats) > limit {
|
||||
chats = chats[:limit]
|
||||
last := chats[len(chats)-1].ID
|
||||
next = &last
|
||||
}
|
||||
if chats == nil {
|
||||
chats = []Chat{}
|
||||
}
|
||||
|
||||
// Догружаем участников.
|
||||
for i := range chats {
|
||||
mrows, err := s.pool.Query(ctx,
|
||||
`SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chats[i].ID)
|
||||
if err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
for mrows.Next() {
|
||||
var uid string
|
||||
if err := mrows.Scan(&uid); err != nil {
|
||||
mrows.Close()
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
chats[i].MemberIDs = append(chats[i].MemberIDs, uid)
|
||||
}
|
||||
mrows.Close()
|
||||
}
|
||||
return chats, next, nil
|
||||
}
|
||||
|
||||
// --- Messages ---
|
||||
|
||||
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (*Message, error) {
|
||||
// Проверка членства + вставка в одной транзакции.
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
|
||||
chatID, senderID,
|
||||
).Scan(&ok); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
var m Message
|
||||
var attach []string
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
|
||||
uuid.NewString(), chatID, senderID, text, []string{},
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
m.AttachmentIDs = attach
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// ListMessages — keyset-пагинация по (created_at, id).
|
||||
// Курсор — это id последнего элемента предыдущей страницы; сам id уникален,
|
||||
// поэтому используем его для поиска (created_at, id) в БД.
|
||||
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cursor string) ([]Message, *string, error) {
|
||||
var rows pgx.Rows
|
||||
var err error
|
||||
|
||||
if cursor == "" {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE chat_id=$1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2`, chatID, limit+1)
|
||||
} else {
|
||||
// Определяем позицию курсора.
|
||||
var cCreated time.Time
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT created_at FROM messages WHERE id=$1 AND chat_id=$2`,
|
||||
cursor, chatID,
|
||||
).Scan(&cCreated); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, ErrInvalid
|
||||
}
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE chat_id=$1
|
||||
AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4`, chatID, cCreated, cursor, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []Message
|
||||
for rows.Next() {
|
||||
var m Message
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
items = append(items, m)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
var next *string
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
last := items[len(items)-1].ID
|
||||
next = &last
|
||||
}
|
||||
if items == nil {
|
||||
items = []Message{}
|
||||
}
|
||||
return items, next, nil
|
||||
}
|
||||
|
||||
func classifyDBError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
switch pgErr.Code {
|
||||
case "57014": // query_canceled (statement_timeout)
|
||||
return ErrTimeout
|
||||
case "55P03": // lock_not_available (lock_timeout)
|
||||
return ErrTimeout
|
||||
case "40001": // serialization_failure
|
||||
return ErrUnavailable
|
||||
case "53300": // too_many_connections
|
||||
return ErrUnavailable
|
||||
case "22P02", "22007", "22023": // invalid_text_representation / invalid_datetime_format / invalid_parameter_value
|
||||
return ErrInvalid
|
||||
}
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return ErrTimeout
|
||||
}
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение
|
||||
// и сохраняет результат идемпотентности в одной транзакции.
|
||||
// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true.
|
||||
func (s *Store) AddMessageIdempotent(
|
||||
ctx context.Context,
|
||||
chatID, senderID, text string,
|
||||
attachmentIDs []string,
|
||||
key, bodyHash string,
|
||||
) (msg *Message, rec *IdempotencyRecord, replay bool, err error) {
|
||||
path := "/api/v1/chats/" + chatID + "/messages"
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
|
||||
if err != nil {
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// 1) Членство.
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
|
||||
chatID, senderID,
|
||||
).Scan(&ok); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, false, ErrNotFound
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// Вложения должны принадлежать этому чату и быть ready.
|
||||
if len(attachmentIDs) > 10 {
|
||||
return nil, nil, false, ErrInvalid
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, aid := range attachmentIDs {
|
||||
if seen[aid] {
|
||||
return nil, nil, false, ErrInvalid
|
||||
}
|
||||
seen[aid] = true
|
||||
var (
|
||||
status string
|
||||
objectKey string
|
||||
)
|
||||
// Вложение из этого же чата; не обязательно ready — клиент увидит картинку,
|
||||
// когда worker завершит обработку. failed — принимаем: сообщение уже создано,
|
||||
// UI покажет «failed» через GET /attachments/{id}.
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT status, object_key FROM attachments WHERE id=$1 AND chat_id=$2`,
|
||||
aid, chatID).Scan(&status, &objectKey); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, false, ErrNotFound
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
if status == AttachmentFailed {
|
||||
// Нельзя прикреплять заведомо неудачные вложения — иначе UI будет
|
||||
// навсегда показывать сломанную картинку без пути к replay.
|
||||
return nil, nil, false, ErrInvalid
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Уже был такой ключ?
|
||||
var storedHash string
|
||||
var storedStatus int
|
||||
var storedResp json.RawMessage
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
|
||||
senderID, path, key,
|
||||
).Scan(&storedHash, &storedStatus, &storedResp)
|
||||
switch {
|
||||
case err == nil:
|
||||
if storedHash != bodyHash {
|
||||
return nil, nil, false, ErrConflict
|
||||
}
|
||||
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
|
||||
case !errors.Is(err, pgx.ErrNoRows):
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// 3) Вставка сообщения.
|
||||
var m Message
|
||||
var attach []string
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
|
||||
uuid.NewString(), chatID, senderID, text, attachmentIDs,
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
|
||||
if err != nil {
|
||||
// Гонка на одном ключе в разных репликах: unique violation в idempotency_keys
|
||||
// маловероятен здесь, но serialization failure возможен.
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
m.AttachmentIDs = attach
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
|
||||
// 4) Сохранение ответа.
|
||||
respJSON, _ := json.Marshal(m)
|
||||
_, err = tx.Exec(ctx,
|
||||
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
|
||||
VALUES ($1,'POST',$2,$3,$4,201,$5)`,
|
||||
senderID, path, key, bodyHash, respJSON)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
// Кто-то параллельно успел вставить: перечитываем в этой же tx.
|
||||
if err2 := tx.QueryRow(ctx,
|
||||
`SELECT request_hash, status_code, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
|
||||
senderID, path, key,
|
||||
).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil {
|
||||
if storedHash != bodyHash {
|
||||
return nil, nil, false, ErrConflict
|
||||
}
|
||||
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
|
||||
}
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
return &m, nil, false, nil
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type S3Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
tr *http.Transport
|
||||
}
|
||||
|
||||
func NewS3Store(ctx context.Context, endpoint, region, bucket, access, secret string, maxBytes, maxPixels int64) (*S3Store, error) {
|
||||
if endpoint == "" || access == "" || secret == "" || bucket == "" {
|
||||
return nil, errors.New("s3 configuration incomplete")
|
||||
}
|
||||
|
||||
tr := &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 100,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 3 * time.Second, // Быстрый отказ при недоступности
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
ResponseHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
cli, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""),
|
||||
Secure: strings.HasPrefix(endpoint, "https://"),
|
||||
Region: region,
|
||||
Transport: tr,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := cli.BucketExists(pingCtx, bucket); err != nil {
|
||||
return nil, fmt.Errorf("s3 bucket check: %w", err)
|
||||
}
|
||||
return &S3Store{client: cli, bucket: bucket, maxBytes: maxBytes, maxPixels: maxPixels, tr: tr}, nil
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// PutValidatedImage читает тело потоково во временный файл на диске,
|
||||
// проверяет размер, MIME, пиксели и затем стримит в S3.
|
||||
// Это гарантирует, что память не растёт с размером файла.
|
||||
func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body io.Reader) (*UploadResult, error) {
|
||||
// 1. Создаём временный файл.
|
||||
tmpFile, err := os.CreateTemp("", "upload-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmpPath := tmpFile.Name()
|
||||
defer func() {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpPath)
|
||||
}()
|
||||
|
||||
// 2. Копируем тело в файл с ограничением.
|
||||
limited := io.LimitReader(body, s.maxBytes+1)
|
||||
size, err := io.Copy(tmpFile, limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if size > s.maxBytes {
|
||||
return nil, ErrTooLarge
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
|
||||
// 3. Проверяем MIME по первым 512 байтам.
|
||||
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
header := make([]byte, 512)
|
||||
n, err := io.ReadFull(tmpFile, header)
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
header = header[:n]
|
||||
ct := http.DetectContentType(header)
|
||||
if ct != "image/png" && ct != "image/jpeg" {
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
|
||||
// 4. Проверяем заголовок изображения (размеры).
|
||||
// DecodeConfig читает только начало, но мы используем файл, поэтому это безопасно.
|
||||
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, _, err := image.DecodeConfig(tmpFile)
|
||||
if err != nil {
|
||||
return nil, ErrNotImage
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > s.maxPixels {
|
||||
return nil, ErrNotImage
|
||||
}
|
||||
|
||||
// 5. Считаем SHA-256 и стримим файл в S3.
|
||||
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var hasher hash.Hash = sha256.New()
|
||||
tee := io.TeeReader(tmpFile, hasher)
|
||||
|
||||
putCtx, cancelPut := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancelPut()
|
||||
|
||||
_, err = s.client.PutObject(putCtx, s.bucket, objectKey, tee, size,
|
||||
minio.PutObjectOptions{ContentType: ct})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sha := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
return &UploadResult{
|
||||
ObjectKey: objectKey,
|
||||
ContentType: ct,
|
||||
SizeBytes: size,
|
||||
SHA256: sha,
|
||||
Width: cfg.Width,
|
||||
Height: cfg.Height,
|
||||
}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
StagingPrefix = "staging/"
|
||||
FinalPrefix = "final/"
|
||||
)
|
||||
|
||||
// MoveToFinal: копирование staging→final и удаление staging.
|
||||
// Используем CopyObject, чтобы не тянуть байты в память. Если MinIO не умеет
|
||||
// server-side copy — используйте compose? MinIO умеет S3 CopyObject.
|
||||
func (s *S3Store) MoveToFinal(ctx context.Context, stagingKey, finalKey string) error {
|
||||
src := minio.CopySrcOptions{Bucket: s.bucket, Object: stagingKey}
|
||||
dst := minio.CopyDestOptions{Bucket: s.bucket, Object: finalKey}
|
||||
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.DeleteObject(ctx, stagingKey)
|
||||
}
|
||||
|
||||
func (s *S3Store) PutBytes(ctx context.Context, key, contentType string, data []byte) (string, error) {
|
||||
h := sha256.Sum256(data)
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h[:]), nil
|
||||
}
|
||||
|
||||
// FetchStagingForProcessing — читает staging в память с жёстким лимитом.
|
||||
// maxBytes гарантирует, что worker не съест RAM на «злом» файле.
|
||||
func (s *S3Store) FetchStagingForProcessing(ctx context.Context, key string) ([]byte, string, error) {
|
||||
rc, ct, size, err := s.GetObject(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
if size > s.maxBytes {
|
||||
return nil, "", ErrTooLarge
|
||||
}
|
||||
buf, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf, ct, nil
|
||||
}
|
||||
|
||||
func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) {
|
||||
// Общий дедлайн на все попытки — 6 секунд.
|
||||
overallCtx, overallCancel := context.WithTimeout(ctx, 6*time.Second)
|
||||
defer overallCancel()
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
s.tr.CloseIdleConnections()
|
||||
}
|
||||
|
||||
attemptCtx, cancel := context.WithTimeout(overallCtx, 3*time.Second)
|
||||
obj, err := s.client.GetObject(attemptCtx, s.bucket, objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
cancel()
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
|
||||
return nil, "", 0, ErrNotFound
|
||||
}
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(obj)
|
||||
obj.Close()
|
||||
cancel()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if int64(len(data)) != info.Size {
|
||||
lastErr = fmt.Errorf("size mismatch: got %d, expected %d", len(data), info.Size)
|
||||
continue
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(data)), info.ContentType, info.Size, nil
|
||||
}
|
||||
|
||||
s.tr.CloseIdleConnections()
|
||||
return nil, "", 0, fmt.Errorf("s3 get failed after retries: %w", lastErr)
|
||||
}
|
||||
|
||||
func (s *S3Store) DeleteObject(ctx context.Context, objectKey string) error {
|
||||
return s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{})
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type SessionStore struct {
|
||||
store *Store
|
||||
cache *SessionCache
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func NewSessionStore(store *Store, cache *SessionCache, ttl time.Duration) *SessionStore {
|
||||
return &SessionStore{store: store, cache: cache, ttl: ttl}
|
||||
}
|
||||
|
||||
func newToken() (string, error) {
|
||||
b := make([]byte, 32) // 256 бит
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// Create — пишем сессию в БД (источник истины) и кэшируем публичные данные User.
|
||||
func (s *SessionStore) Create(ctx context.Context, user User) (*Session, error) {
|
||||
token, err := newToken()
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
expires := time.Now().UTC().Add(s.ttl)
|
||||
if _, err := s.store.pool.Exec(ctx,
|
||||
`INSERT INTO sessions (token, user_id, expires_at) VALUES ($1,$2,$3)`,
|
||||
token, user.ID, expires); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
if s.cache != nil {
|
||||
s.cache.SetUser(ctx, user.ID, user, expires)
|
||||
}
|
||||
return &Session{
|
||||
Token: token,
|
||||
TokenType: "Bearer",
|
||||
ExpiresAt: expires,
|
||||
User: user,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Lookup ВСЕГДА обращается к PostgreSQL за фактом существования сессии.
|
||||
// Кэш используется только для публичных данных User.
|
||||
func (s *SessionStore) Lookup(ctx context.Context, token string) (*User, error) {
|
||||
var (
|
||||
userID string
|
||||
expiresAt time.Time
|
||||
)
|
||||
err := s.store.pool.QueryRow(ctx,
|
||||
`SELECT user_id, expires_at FROM sessions WHERE token=$1`, token,
|
||||
).Scan(&userID, &expiresAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrUnauthorized
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
if time.Now().UTC().After(expiresAt) {
|
||||
_, _ = s.store.pool.Exec(ctx, `DELETE FROM sessions WHERE token=$1`, token)
|
||||
return nil, ErrUnauthorized
|
||||
}
|
||||
|
||||
if s.cache != nil {
|
||||
if u, ok := s.cache.GetUser(ctx, userID); ok {
|
||||
return &u, nil
|
||||
}
|
||||
}
|
||||
|
||||
u, err := s.store.GetUserByID(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, ErrUnauthorized
|
||||
}
|
||||
if s.cache != nil {
|
||||
s.cache.SetUser(ctx, userID, *u, expiresAt)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (s *SessionStore) Revoke(ctx context.Context, token string) error {
|
||||
var userID string
|
||||
err := s.store.pool.QueryRow(ctx,
|
||||
`DELETE FROM sessions WHERE token=$1 RETURNING user_id`, token,
|
||||
).Scan(&userID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// Сессии уже нет — с точки зрения клиента токен отозван.
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if s.cache != nil {
|
||||
// Кэш User не критичен, но полезно сбросить.
|
||||
_ = s.cache.DelUser(ctx, userID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""В Stream нет filename/URL/секретов — только id/version/attempt."""
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
upload(c, a, ch, content)
|
||||
raw = docker('exec', '-T', 'redis', 'redis-cli', '--raw', 'XRANGE', 'attachment.jobs', '-', '+')
|
||||
ok('stream sample:\n' + raw)
|
||||
lower = raw.lower()
|
||||
for bad in ('filename', 'sample.png', 'staging/', 'final/', 'http', 'secret'):
|
||||
check(bad not in lower, 'broker payload contains forbidden token: %s' % bad)
|
||||
ok('broker payload has only ids/version/attempt')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Shared helpers for student lab5 tests; no tokens are logged."""
|
||||
import base64, json, os, subprocess, sys, time, uuid
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, Failure, check # noqa: E402
|
||||
|
||||
def fail(msg):
|
||||
print('FAIL:', msg)
|
||||
raise SystemExit(1)
|
||||
|
||||
def ok(msg='PASS'):
|
||||
print(msg)
|
||||
|
||||
def docker(*args, check_rc=True):
|
||||
"""Run docker compose <args>; return stdout. Raises if rc!=0 and check_rc."""
|
||||
rc = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True)
|
||||
if check_rc and rc.returncode != 0:
|
||||
raise Failure('docker compose %s: %s' % (' '.join(args), rc.stderr.strip()))
|
||||
return rc.stdout
|
||||
|
||||
def psql(sql):
|
||||
return docker('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-tAc', sql).strip()
|
||||
|
||||
def upload(c, actor, chat, content, filename='sample.png', ct='image/png'):
|
||||
return c.call('POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
|
||||
202, actor=actor, raw=content, headers={'Content-Type': ct},
|
||||
schema='Attachment')['json']
|
||||
|
||||
def wait_status(c, actor, att_id, want, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
|
||||
if att['status'] == want:
|
||||
return att
|
||||
if att['status'] == 'failed' and want != 'failed':
|
||||
fail('attachment failed: ' + str(att.get('error_code')))
|
||||
time.sleep(0.25)
|
||||
fail('attachment did not reach %s in %ss' % (want, seconds))
|
||||
|
||||
def png_bytes(w, h, seed=0):
|
||||
"""Generate a valid PNG without external libs.
|
||||
|
||||
Быстро и с малой памятью: одна строка пикселей переиспользуется.
|
||||
Для 7000x7000 занимает ~1-2 секунды и ~150 MiB RAM.
|
||||
"""
|
||||
import struct, zlib
|
||||
|
||||
# одна строка: filter byte 0 + w * 3 байта RGB
|
||||
row = bytearray(1 + w * 3)
|
||||
for x in range(w):
|
||||
row[1 + x*3] = (x * 7 + seed) & 0xFF
|
||||
row[2 + x*3] = (x * 11) & 0xFF
|
||||
row[3 + x*3] = (x * 13) & 0xFF
|
||||
row = bytes(row)
|
||||
|
||||
# zlib потоково, без накопления всего raw
|
||||
co = zlib.compressobj(level=6)
|
||||
idat_parts = []
|
||||
for y in range(h):
|
||||
idat_parts.append(co.compress(row))
|
||||
idat_parts.append(co.flush())
|
||||
idat = b''.join(idat_parts)
|
||||
|
||||
def chunk(t, d):
|
||||
return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff)
|
||||
|
||||
ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)
|
||||
return (b'\x89PNG\r\n\x1a\n'
|
||||
+ chunk(b'IHDR', ihdr)
|
||||
+ chunk(b'IDAT', idat)
|
||||
+ chunk(b'IEND', b''))
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""kill -9 worker после S3 write до ack → повтор не создаёт дублей."""
|
||||
import sys, time, signal
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = png_bytes(4000, 4000)
|
||||
|
||||
# Сначала готовим staging+job, но без запуска worker’а.
|
||||
docker('stop', 'worker')
|
||||
att = upload(c, a, ch, content)
|
||||
# Запускаем worker — он возьмёт processing и, скорее всего, быстро закончит.
|
||||
docker('start', 'worker')
|
||||
# Ловим момент processing и убиваем процесс.
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
if st == 'processing':
|
||||
break
|
||||
time.sleep(0.1)
|
||||
docker('kill', '-s', 'SIGKILL', 'worker')
|
||||
ok('worker killed during processing')
|
||||
|
||||
# После lease (60s) + reaper (30s) job должна снова стать queued.
|
||||
time.sleep(100)
|
||||
docker('start', 'worker')
|
||||
att = wait_status(c, a, att['id'], 'ready', seconds=120)
|
||||
n = int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']))
|
||||
check(n == 3, 'expected exactly 3 variants, got %d' % n)
|
||||
ok('recovery produced exactly 3 non-original variants, no duplicates (original in object_key)')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Реестр failed доступен оператору (SELECT из attachment_jobs)."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
|
||||
att = upload(c, a, ch, content)
|
||||
wait_status(c, a, att['id'], 'failed', seconds=180)
|
||||
rows = psql(
|
||||
"SELECT attachment_id, last_error_code, attempt FROM attachment_jobs "
|
||||
"WHERE status='failed' ORDER BY updated_at DESC LIMIT 5")
|
||||
check(rows, 'no failed jobs visible to operator')
|
||||
ok('failed registry:\n' + rows)
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Битый файл: 5 attempt → failed, error_code=invalid_image."""
|
||||
import sys, time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
# Заведомо валидный PNG в API (MIME проходит), но worker получит
|
||||
# битый заголовок, если подменить staging — здесь проверяем
|
||||
# реалистичный сценарий: JPEG с обрезанным хвостом.
|
||||
content = (Path('tests/fixtures/sample.png')).read_bytes()[:-20]
|
||||
att = upload(c, a, ch, content)
|
||||
# API пропустит (DecodeConfig проходит), worker упадёт на Decode.
|
||||
att = wait_status(c, a, att['id'], 'failed', seconds=180)
|
||||
check(att.get('error_code') == 'invalid_image', 'unexpected error_code: %s' % att.get('error_code'))
|
||||
attempts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
|
||||
# invalid_image — постоянная ошибка, worker не должен ретраить.
|
||||
check(attempts == 1, 'permanent error must not be retried: attempt=%d' % attempts)
|
||||
ok('failed immediately (attempt=1), error_code=invalid_image — permanent, not retried')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Мини-парсер SOF для JPEG; без внешних библиотек."""
|
||||
from tests.client import Failure, check
|
||||
|
||||
|
||||
def jpeg_size(data):
|
||||
check(data[:2] == b'\xff\xd8', 'thumbnail is not JPEG')
|
||||
i = 2
|
||||
sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf}
|
||||
while i < len(data):
|
||||
check(data[i] == 0xff, 'invalid JPEG marker')
|
||||
while i < len(data) and data[i] == 0xff:
|
||||
i += 1
|
||||
check(i < len(data), 'truncated JPEG marker')
|
||||
marker = data[i]
|
||||
i += 1
|
||||
if marker in {0xd9, 0xda}:
|
||||
break
|
||||
if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7:
|
||||
continue
|
||||
check(i + 2 <= len(data), 'truncated JPEG segment')
|
||||
length = int.from_bytes(data[i:i + 2], 'big')
|
||||
check(length >= 2 and i + length <= len(data), 'invalid JPEG segment length')
|
||||
if marker in sof:
|
||||
check(length >= 8, 'invalid JPEG SOF')
|
||||
return (int.from_bytes(data[i + 5:i + 7], 'big'),
|
||||
int.from_bytes(data[i + 3:i + 5], 'big'))
|
||||
i += length
|
||||
raise Failure('JPEG dimensions not found')
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reaper восстанавливает job после паузы worker’а дольше lease (60с)."""
|
||||
import sys, time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = png_bytes(4000, 4000)
|
||||
|
||||
# 1) Готовим job, но worker не даём её взять.
|
||||
docker('pause', 'worker')
|
||||
try:
|
||||
att = upload(c, a, ch, content)
|
||||
finally:
|
||||
pass # unpause ниже — после того, как настроим SIGSTOP
|
||||
|
||||
# 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP.
|
||||
# Используем docker kill --signal=SIGSTOP, чтобы остановить процесс
|
||||
# (не контейнер). Контейнер "жив", но worker заморожен.
|
||||
docker('unpause', 'worker')
|
||||
|
||||
# 3) Ждём processing; как только взял — сразу SIGSTOP.
|
||||
deadline = time.monotonic() + 15
|
||||
job = None
|
||||
while time.monotonic() < deadline:
|
||||
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
if st == 'processing':
|
||||
# Останавливаем процесс, пока он внутри processing.
|
||||
docker('kill', '-s', 'SIGSTOP', 'worker')
|
||||
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
break
|
||||
if st == 'done':
|
||||
fail('job finished too fast to simulate stuck worker')
|
||||
time.sleep(0.02)
|
||||
|
||||
check(job is not None, 'worker never entered processing')
|
||||
|
||||
# 4) Фиксируем lease_expires_at на момент SIGSTOP.
|
||||
lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job)
|
||||
ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease))
|
||||
|
||||
# 5) Ждём, пока lease протухнет и reaper вернёт job в queued.
|
||||
# lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом.
|
||||
deadline = time.monotonic() + 120
|
||||
requeued = False
|
||||
while time.monotonic() < deadline:
|
||||
st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job)
|
||||
if st == 'queued':
|
||||
requeued = True
|
||||
break
|
||||
time.sleep(2)
|
||||
check(requeued, 'reaper did not requeue stuck job (status=%s)' % st)
|
||||
ok('reaper requeued the stuck job')
|
||||
|
||||
# 6) Возобновляем worker: он должен доделать job до ready.
|
||||
docker('kill', '-s', 'SIGCONT', 'worker')
|
||||
att = wait_status(c, a, att['id'], 'ready', seconds=120)
|
||||
ok('ready after unstuck')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""413, 415, 400 invalid_image — лимиты bytes/pixels/MIME."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import png_bytes, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
url = '/api/v1/chats/' + ch['id'] + '/attachments?filename=x.bin'
|
||||
|
||||
# 413: > 10 MiB
|
||||
big = b'\x89PNG\r\n\x1a\n' + b'\x00' * (11 * 1024 * 1024)
|
||||
r = c.call('POST', url, 413, actor=a, raw=big, headers={'Content-Type': 'image/png'})
|
||||
ok('413 payload_too_large')
|
||||
|
||||
# 415: text/plain
|
||||
r = c.call('POST', url, 415, actor=a, raw=b'hello', headers={'Content-Type': 'text/plain'})
|
||||
ok('415 unsupported_media_type')
|
||||
|
||||
# 415: клиент говорит image/png, но содержимое не PNG/JPEG по DetectContentType
|
||||
r = c.call('POST', url, 415, actor=a, raw=b'\x00' * 100, headers={'Content-Type': 'image/png'})
|
||||
ok('415 mismatched content (client MIME not trusted)')
|
||||
|
||||
# 400: PNG-сигнатура валидна (DetectContentType -> image/png),
|
||||
# но IHDR/IDAT битые -> DecodeConfig падает -> invalid_image (400)
|
||||
broken_png = b'\x89PNG\r\n\x1a\n' + b'\x00' * 40
|
||||
r = c.call('POST', url, 400, actor=a, raw=broken_png, headers={'Content-Type': 'image/png'})
|
||||
ok('400 invalid image (signature ok, header broken)')
|
||||
|
||||
# 400: пиксели > MAX_IMAGE_PIXELS (если выставите 1e8 — этот тест не сработает;
|
||||
# по умолчанию 40 MP, значит нужно 7000x7000 ≈ 49 MP)
|
||||
huge = png_bytes(7000, 7000)
|
||||
r = c.call('POST', url, 400, actor=a, raw=huge, headers={'Content-Type': 'image/png'})
|
||||
ok('400 too many pixels')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Outbox: пауза outbox → 202 → unpause → ready; published_at=NULL в промежутке."""
|
||||
import sys, time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
docker('pause', 'outbox')
|
||||
try:
|
||||
att = upload(c, a, ch, content)
|
||||
check(att['status'] == 'queued', 'expected queued')
|
||||
time.sleep(3)
|
||||
published = psql("SELECT coalesce(published_at::text,'NULL') FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
check(published == 'NULL', 'outbox paused but published_at=%s' % published)
|
||||
ok('outbox paused: published_at IS NULL, upload accepted')
|
||||
finally:
|
||||
docker('unpause', 'outbox')
|
||||
|
||||
att = wait_status(c, a, att['id'], 'ready', seconds=90)
|
||||
ok('after unpause: ready')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Повторный XADD того же job_id не создаёт дублей вариантов."""
|
||||
import json, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
att = upload(c, a, ch, content)
|
||||
att = wait_status(c, a, att['id'], 'ready')
|
||||
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
|
||||
check(before == '3', 'expected 3 variants, got %s' % before)
|
||||
|
||||
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
payload = json.dumps({
|
||||
"schema_version": 1, "job_id": job, "type": "attachment.process.v1",
|
||||
"attachment_id": att['id'], "pipeline_version": 1, "attempt": 1,
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
})
|
||||
# Повторная доставка через Redis CLI.
|
||||
docker('exec', '-T', 'redis', 'redis-cli', 'XADD', 'attachment.jobs', '*', 'payload', payload)
|
||||
time.sleep(5)
|
||||
after = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
|
||||
check(after == '3', 'redelivery created duplicate variants: %s' % after)
|
||||
ok('redelivery is no-op (still 3 non-original variants; original via object_key)')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay failed → снова queued."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
# Форсим failed (битый файл проходит DecodeConfig, но не Decode).
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
|
||||
att = upload(c, a, ch, content)
|
||||
att = wait_status(c, a, att['id'], 'failed', seconds=180)
|
||||
ok('failed reached')
|
||||
|
||||
c.call('POST', '/api/v1/attachments/' + att['id'] + '/replay', 202, actor=a)
|
||||
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
check(st == 'queued', 'after replay expected queued, got %s' % st)
|
||||
ok('replay -> queued')
|
||||
|
||||
# Replay дал одну новую попытку; ошибка permanent, поэтому снова failed при attempt=1.
|
||||
att = wait_status(c, a, att['id'], 'failed', seconds=180)
|
||||
atts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
|
||||
check(atts == 1, 'after replay: permanent error must fail at attempt=1, got %d' % atts)
|
||||
ok('replay reset attempt to 1; permanent error failed again — not retried')
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready."""
|
||||
import subprocess, sys, time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
# 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем.
|
||||
root_user = open('secrets/minio_root_user.txt').read().strip()
|
||||
root_pass = open('secrets/minio_root_password.txt').read().strip()
|
||||
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc',
|
||||
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
|
||||
|
||||
# 2) Пауза worker’а: job не будет взят до unpause.
|
||||
docker('pause', 'worker')
|
||||
try:
|
||||
att = upload(c, a, ch, content)
|
||||
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
|
||||
check(staging.startswith('staging/'), 'unexpected staging_key')
|
||||
|
||||
# 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект.
|
||||
# Если worker ещё не стартовал — GetObject вернёт NoSuchKey.
|
||||
subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm',
|
||||
'loc/messenger-attachments/' + staging],
|
||||
capture_output=True, text=True, check=True)
|
||||
|
||||
# 4) Отдаём job worker’у.
|
||||
docker('unpause', 'worker')
|
||||
|
||||
# 5) Ждём, пока attempt >= 2.
|
||||
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||||
deadline = time.monotonic() + 90
|
||||
attempt = 1
|
||||
while time.monotonic() < deadline:
|
||||
attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job))
|
||||
if attempt >= 2:
|
||||
break
|
||||
time.sleep(1)
|
||||
check(attempt >= 2,
|
||||
'retry did not happen on missing staging object (attempt=%d)' % attempt)
|
||||
ok('transient error retried: attempt=%d' % attempt)
|
||||
|
||||
# 6) Логи retry именно для нашего job_id.
|
||||
logs = docker('logs', '--tail=500', 'worker')
|
||||
lines = [l for l in logs.splitlines()
|
||||
if ('retry job=' + job) in l]
|
||||
check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:]))
|
||||
ok('retry log lines:')
|
||||
for l in lines:
|
||||
print(' ' + l)
|
||||
|
||||
# 7) Возвращаем staging — job должен пройти retry и стать ready.
|
||||
# Перезаписываем объект с правильным содержимым.
|
||||
# Через mc pipe: передаём байты на stdin.
|
||||
r = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 's3',
|
||||
'mc', 'pipe', 'loc/messenger-attachments/' + staging],
|
||||
input=content, capture_output=True)
|
||||
check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode())
|
||||
|
||||
att = wait_status(c, a, att['id'], 'ready', seconds=180)
|
||||
ok('recovered to ready after staging restored (attempt=%d)' % attempt)
|
||||
|
||||
finally:
|
||||
# На всякий случай — если упали в середине, worker не должен остаться на паузе.
|
||||
subprocess.run(['docker', 'compose', 'unpause', 'worker'],
|
||||
capture_output=True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""После ready staging пуст (объект перенесён в final)."""
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, psql, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
att = upload(c, a, ch, content)
|
||||
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
|
||||
check(staging.startswith('staging/'), 'staging_key does not start with staging/')
|
||||
att = wait_status(c, a, att['id'], 'ready')
|
||||
root_user = open('secrets/minio_root_user.txt').read().strip()
|
||||
root_pass = open('secrets/minio_root_password.txt').read().strip()
|
||||
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'local_check',
|
||||
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
|
||||
|
||||
r = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'stat',
|
||||
'local_check/messenger-attachments/' + staging],
|
||||
capture_output=True, text=True)
|
||||
combined = ((r.stdout or '') + (r.stderr or '')).lower()
|
||||
check('does not exist' in combined or 'not found' in combined,
|
||||
'staging object still present: %s\n%s' % (staging, combined))
|
||||
ok('staging object removed after ready (mc rc=%d)' % r.returncode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""queued → processing → ready через остановку worker’а."""
|
||||
import time, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, docker, ok, fail
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
docker('pause', 'worker')
|
||||
try:
|
||||
att = upload(c, a, ch, content)
|
||||
check(att['status'] == 'queued', 'expected queued after upload')
|
||||
ok('upload -> queued')
|
||||
# Не должно перейти в processing, пока worker на паузе.
|
||||
time.sleep(3)
|
||||
att = c.call('GET', '/api/v1/attachments/' + att['id'], actor=a, schema='Attachment')['json']
|
||||
check(att['status'] == 'queued', 'worker paused but status changed to %s' % att['status'])
|
||||
ok('status stays queued while worker paused')
|
||||
finally:
|
||||
docker('unpause', 'worker')
|
||||
|
||||
att = wait_status(c, a, att['id'], 'ready', seconds=90)
|
||||
names = {v['name'] for v in att['variants']}
|
||||
# original в variants[] нет по контракту OpenAPI — он в object_key/оригинальном файле.
|
||||
check(names == {'thumbnail', 'crop', 'optimized'},
|
||||
'variants mismatch: %s' % names)
|
||||
ok('ready with thumbnail/crop/optimized (original available via ?variant=original)')
|
||||
|
||||
# original доступен отдельно — проверим, что эндпоинт отвечает 200.
|
||||
r = c.call('GET',
|
||||
'/api/v1/attachments/' + att['id'] + '/content?variant=original',
|
||||
200, actor=a)
|
||||
check(len(r['bytes']) > 0, 'original empty')
|
||||
ok('original served via ?variant=original (%d bytes)' % len(r['bytes']))
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""original неизменен, thumbnail 256x256, crop 128x128, optimized существует."""
|
||||
import hashlib, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from tests.client import Client, check
|
||||
from student_tests.common import upload, wait_status, ok, fail
|
||||
from student_tests.jpeg_probe import jpeg_size as dims_jpeg
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
a, b, e, ch = c.bootstrap()
|
||||
content = Path('tests/fixtures/sample.png').read_bytes()
|
||||
att = upload(c, a, ch, content)
|
||||
att = wait_status(c, a, att['id'], 'ready')
|
||||
src_sha = hashlib.sha256(content).hexdigest()
|
||||
|
||||
def get(v):
|
||||
return c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (att['id'], v),
|
||||
200, actor=a)['bytes']
|
||||
|
||||
orig = get('original')
|
||||
check(hashlib.sha256(orig).hexdigest() == src_sha, 'original changed')
|
||||
ok('original byte-identical')
|
||||
|
||||
th = get('thumbnail')
|
||||
w, h = dims_jpeg(th)
|
||||
check(max(w, h) == 256 and w <= 256 and h <= 256,
|
||||
'thumbnail is %dx%d, expected fit in 256x256' % (w, h))
|
||||
check(w * 160 == h * 256 if h == 160 else True, ...) # необязательно
|
||||
# Проще всего — сверить с metadata:
|
||||
thumb_meta = next(v for v in att['variants'] if v['name'] == 'thumbnail')
|
||||
check((w, h) == (thumb_meta['width'], thumb_meta['height']),
|
||||
'thumbnail metadata %sx%s != actual %dx%d' % (thumb_meta['width'], thumb_meta['height'], w, h))
|
||||
ok('thumbnail %dx%d (metadata matches)' % (w, h))
|
||||
|
||||
cr = get('crop')
|
||||
w, h = dims_jpeg(cr)
|
||||
check((w, h) == (128, 128), 'crop is %dx%d' % (w, h))
|
||||
ok('crop 128x128 JPEG')
|
||||
|
||||
opt = get('optimized')
|
||||
check(len(opt) > 0, 'optimized empty')
|
||||
# optimized может быть JPEG или WebP; проверяем, что не пусто и похоже на изображение
|
||||
check(opt[:2] == b'\xff\xd8' or opt[:4] == b'RIFF', 'optimized is not JPEG/WebP')
|
||||
ok('optimized present (%d bytes)' % len(opt))
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
fail(str(exc))
|
||||
Reference in New Issue
Block a user