Author SHA1 Message Date
Оля Мороз c91ec7790f feat:report 2026-11-16 20:30:12 +03:00
Оля Мороз 90f407b174 feat:level4 2026-11-16 14:41:07 +03:00
Оля Мороз 0a361367c7 feat:lab4-catchup 2026-11-16 11:46:03 +03:00
53 changed files with 5014 additions and 20 deletions
+32
View File
@@ -0,0 +1,32 @@
# --- build stage ---
FROM golang:1.26-alpine AS build
WORKDIR /src
RUN apk add --no-cache ca-certificates
COPY go.mod go.sum ./
RUN go mod download
COPY cmd ./cmd
COPY src ./src
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger && \
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/outbox ./cmd/outbox && \
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker
# --- runtime stage ---
FROM alpine:3.20
RUN apk add --no-cache ca-certificates postgresql-client && \
addgroup -S app && adduser -S -G app app
WORKDIR /app
COPY --from=build /out/messenger /app/messenger
COPY --from=build /out/worker /app/worker
COPY --from=build /out/outbox /app/outbox
COPY migrations /app/migrations
COPY scripts/migrate.sh /app/migrate.sh
RUN chmod +x /app/migrate.sh
USER app
EXPOSE 8080
ENTRYPOINT ["/app/messenger"]
+22 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover my_tests my_tests_quick my_tests_slow
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,24 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# Быстрые (<1 мин): статусы, лимиты, варианты, payload, DLQ, staging.
my_tests_quick:
python3 student_tests/states.py
python3 student_tests/limits.py
python3 student_tests/variants.py
python3 student_tests/broker-payload.py
python3 student_tests/staging-cleanup.py
python3 student_tests/dlq.py
python3 student_tests/redelivery.py
# Долгие (>2 мин): retry/backoff/failed/replay/outbox/lease/crash.
my_tests_slow:
python3 student_tests/outbox.py
python3 student_tests/failed.py
python3 student_tests/retry.py
python3 student_tests/replay.py
python3 student_tests/lease.py
python3 student_tests/crash-before-ack.py
my_tests: my_tests_quick my_tests_slow
+166 -19
View File
@@ -1,37 +1,184 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Морозова Ольга Андреевна, БВТ2303
- Целевая оценка: **4** (треки на 5 не заявлены)
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники: [README.md](README.md), [COURSE.md](COURSE.md), [contracts/openapi.json](contracts/openapi.json), документация Go, MinIO (mc), PostgreSQL, Redis Streams, Nginx, Docker.
- Библиотеки: Go 1.26 (stdlib), github.com/google/uuid v1.6.0, github.com/jackc/pgx/v5 v5.5.5, github.com/redis/go-redis/v9 v9.5.1, golang.org/x/crypto v0.23.0, github.com/minio/minio-go/v7 v7.0.72, golang.org/x/image v0.48.0. Тесты — Python 3 (stdlib).
- Помощь ИИ: Deepseek — написание и проверка кода.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Команды выполнялись в WSL2 (Ubuntu) на Windows 11. Требования: Docker 24+ (Compose v2), Python 3.9+ (для тестов), `make`.
Версии (фиксированные теги образов):
- `golang:1.26-alpine` — build stage
- `alpine:3.20` — runtime stage
- `postgres:16.3-alpine` — база данных
- `redis:7.2.5-alpine` — кэш сессий и брокер (Redis Streams)
- `nginx:1.27.0-alpine` — балансировщик
- `pgsty/silo:RELEASE.2026-10-26T00-00-00Z` — S3-совместимое хранилище (MinIO-compatible)
### Подготовка
```sh
git clone https://gitea.privcloud.tech/MiniLynx/mtusi2026lab5-fork
cd mtusi2026lab5-fork
# 1. Секреты (не входят в Git)
mkdir -p secrets
printf 'local-dev-postgres-password\n' > secrets/postgres_password.txt
printf 'postgres://messenger:local-dev-postgres-password@postgres:5432/messenger?sslmode=disable\n' > secrets/database_url.txt
printf 'redis://redis:6379/0\n' > secrets/cache_url.txt
printf 'minioadmin\n' > secrets/minio_root_user.txt
printf 'minioadmin-dev-password\n' > secrets/minio_root_password.txt
printf 'messenger-api\n' > secrets/s3_access_key_id.txt
printf 'messenger-api-dev-secret\n' > secrets/s3_secret_access_key.txt
chmod 600 secrets/*.txt
# 2. Локальный .env
cp .env.example .env
# ВАЖНО: В .env нужно исправить S3_ENDPOINT на s3:9000 (без http://)
sed -i 's|S3_ENDPOINT=.*|S3_ENDPOINT=s3:9000|' .env
# 3. Python для проверок
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements-dev.txt
# 4. Скрипты
chmod +x scripts/migrate.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh
```
### Сборка и запуск
```sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождаться /health/ready
docker compose ps
```
Для запуска с другим количеством реплик добавить `API_REPLICAS=$n` перед `docker compose --env-file .env up -d --build`, где `$n` - желаемое количество реплик.
### Тесты
```sh
make check
make validate PYTHON=.venv/bin/python
make test
# Persistence
make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Failover
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
./scripts/restore-api.sh
# Собственные тесты (pipeline, сбои, лимиты)
make my_tests
```
## Остановка (с сохранением volumes)
```sh
docker compose --env-file .env down --remove-orphans
```
### Намеренное удаление учебных данных
```sh
docker compose --env-file .env down --remove-orphans -v
rm -f secrets/*.txt
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
HTTP-клиент → Nginx (8080) → API (Go, x2, stateless)
│
├──[S3 PUT]→ staging/ (S3, приватный bucket)
├──[INSERT]→ Postgres: attachments (queued) + attachment_jobs (outbox)
└──[202 Accepted]
Outbox (x1) ⇄ Postgres (attachment_jobs.published_at IS NULL)
│
└──[XADD]→ Redis Streams (attachment.jobs)
Worker (x2) ⇄ Redis Streams (consumer group)
├──[SELECT ... FOR UPDATE SKIP LOCKED]→ Postgres (lease 60s)
├──[GET]→ staging/ → resize (thumb 256, crop 128, opt) →[PUT]→ final/
├──[INSERT variants + UPDATE ready]→ Postgres (транзакция)
└──[XACK]→ Redis
```
- **Асинхронный приём (уровень 3):** `POST /api/v1/chats/{id}/attachments` возвращает **202 Accepted** сразу после того, как файл сохранён в staging, а в БД одной транзакцией записаны метаданные `attachments` (status=`queued`) и `attachment_jobs` (outbox). Это гарантирует, что после ответа 202 работа не потеряется при рестарте API.
- **Outbox (уровень 4):** отдельный процесс `cmd/outbox` читает `attachment_jobs WHERE published_at IS NULL` и публикует в брокер. `published_at` ставится только после `XADD`. Падение между COMMIT и publish приводит к повторной публикации при следующем цикле; worker идемпотентен.
- **Брокер:** Redis Streams (тот же Redis, AOF `--appendonly yes`). Consumer Group `workers`, `XACK` после durable-эффекта. В сообщении только `job_id`, `attachment_id`, `pipeline_version`, `attempt` (никаких URL или filename).
- **Worker (уровень 3-4):** отдельный процесс, реплики 2. Забирает job через `ClaimJobByID` (lease 60s). Пишет `original` (server-side copy staging→final), `thumbnail` (256×256, fit), `crop` (128×128, center-cover) и `optimized` (JPEG q=70). Всё в одной транзакции с `attachment_variants` и `status='ready'`. `XACK` только после COMMIT.
- **Идемпотентность:** ключ эффекта — `(attachment_id, pipeline_version, variant)`. Повторная доставка того же job_id: `MarkReady` — no-op (если `status='done'`), `attachment_variants` — `ON CONFLICT DO NOTHING`. Также `VariantsExist` перед Claim.
- **Retry:** `RetryJob` различает permanent (`invalid_image` → сразу failed) и transient (`storage_error` → backoff `base * 2^(n-1) + jitter`, max 5 попыток). Финальный `failed` с безопасным `error_code`.
- **Зависшие job:** `RequeueStuck` в outbox и worker, каждые 30с: `status='processing' AND lease_expires_at < now()` → `queued`, `published_at=NULL`.
- **Replay:** `POST /api/v1/attachments/{id}/replay` сбрасывает `attempt=1`, `status='queued'`, `published_at=NULL` для failed-задач.
- **Очистка:** `MoveToFinal` удаляет staging после CopyObject. `ReconcileOrphans` (в API) чистит `attachment_gc` и «зависшие» staging-объекты без job.
### Что перенесено из lab4
- PostgreSQL, Argon2id, sessions (с кэшем в Redis), горизонтальное масштабирование, Nginx.
- Механизм идемпотентности `idempotency_keys` для сообщений.
- S3-хранилище и приватный bucket, `S3Store` с таймаутами и ретраями.
### Что изменено
- Миграция `0002_attachments.sql`: добавлены `attachment_variants`, `attachment_jobs` (outbox+lease), поля `staging_key`, `pipeline_version`, `ready_at`, `failed_at`. Таблица `attachments` теперь содержит `status IN ('queued','processing','ready','failed')`.
- `UploadAttachment` возвращает 202, статус `queued`; `DownloadAttachment` → 409 для не-ready.
- Добавлены `cmd/outbox`, `cmd/worker` (обработка изображений, `process.go`).
- `src/domain/broker.go` — обёртка Redis Streams.
- `compose.yaml`: сервисы `outbox` (1 реплика) и `worker` (2 реплики), миграции вынесены в отдельный сервис `migrate`.
- В `attachments.go`: `EnqueueAttachment`, `ClaimJob`, `ClaimJobByID`, `MarkReady`, `RetryJob`, `RequeueStuck`, `ReplayFailed`, `MarkFailedPermanent`, `GetVariantKey`, `SelectUnpublishedJobs`, `MarkPublished`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------ |
| Offline-структура и локальные refs | `make check` | PASS | [png](evidence/2_Tests_Passing.png) |
| Полная OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | [png](evidence/2_Tests_Passing.png) |
| Публичный smoke (lab5) | `make test` | PASS: 202, ready, thumbnail | [png](evidence/2_Tests_Passing.png) |
| Persistence: перезапуск API+БД+Redis+S3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | [png](evidence/2_Tests_Passing.png) |
| Failover: остановка одной реплики API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | `observed_instances_before: 2`, `successful_writes: 75`, `first_recovery: 5.371s` | [png](evidence/2_Tests_Passing.png) |
| **Ур. 3:** Upload → 202 → queued (worker не нужен) | `student_tests/states.py` | queued → ready, 3 варианта + original через `?variant=original` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Broker и worker — отдельные процессы | `docker compose ps` | `outbox`, `worker-1`, `worker-2` | [png](evidence/1_Healthcheck.png) |
| **Ур. 3:** Статусы в API, лимиты и типы | `student_tests/limits.py` | 413, 415, 400 invalid_image | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** original неизменен, thumbnail по контракту | `student_tests/variants.py` | `byte-identical`, `thumbnail 256x160 (metadata matches)`, `crop 128x128` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Broker payload без filename/URL | `student_tests/broker-payload.py` | только id/version/attempt | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Очистка staging после ready | `student_tests/staging-cleanup.py` | `staging object removed after ready` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Реестр failed (DLQ) | `student_tests/dlq.py` | `invalid_image\|1` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Идемпотентность при повторной доставке | `student_tests/redelivery.py` | `still 3 non-original variants` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Outbox закрывает разрыв COMMIT→publish | `student_tests/outbox.py` | `published_at IS NULL` при паузе outbox → ready после unpause | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Permanent error без retry | `student_tests/failed.py` | `attempt=1, invalid_image — not retried` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Transient error → retry → ready | `student_tests/retry.py` | `attempt=2`, `retry log lines: ... backoff=2s jitter=451ms` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Replay failed | `student_tests/replay.py` | `replay -> queued`, повторный permanent fail | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Lease / reaper зависших | `student_tests/lease.py` | `reaper requeued the stuck job` → `ready` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Crash после side effect до ack | `student_tests/crash-before-ack.py` | `recovery produced exactly 3 variants, no duplicates` | [txt](evidence/my_tests_passing.txt) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- |
| Остановка одной реплики API | 0 потерянных подтверждённых записей, LB переключается | Failover PASS, 75 успешных сообщений | 5.371 с, потерь нет |
| Рестарт API/БД/Redis/S3 (volumes) | Вложение и сессия живы | Persistence PASS | ~30 с (запуск контейнеров), потерь нет |
| Отказ S3 | API возвращает 503, worker ретраит | `retry.py` PASS, `attempt=2`, backoff сработал | После восстановления S3 — ready |
| Пауза outbox (разрыв COMMIT→publish) | `published_at=NULL`, upload 202 | `outbox.py` PASS, job дошёл до ready после unpause | Задержка publish = период паузы, потерь нет |
| Пауза worker (зависший processing) | Lease истекает, reaper возвращает в queued | `lease.py` PASS, job вернулся в queued через ~60с + reaper | ~90 с, потерь нет |
| `SIGKILL` worker во время processing | Повторная доставка, без дублей вариантов | `crash-before-ack.py` PASS, 3 варианта, без дублей | ~100 с (lease + рестарт), потерь нет |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **At-least-once:** доставка Redis Streams + идемпотентный эффект ≠ exactly-once. При падении между XADD и MarkPublished возможна повторная публикация; повторный worker это корректно обрабатывает, но сообщение в Stream продублируется.
- **Replay** доступен через endpoint `POST /api/v1/attachments/{id}/replay` (расширение сверх OpenAPI). В отчёте зафиксировано, что это не меняет семантику обязательных путей.
- **Optimized** — JPEG q=70 вместо WebP (WebP-кодирование требует cgo-зависимостей, которых в alpine-сборке без CGO нет). Контракт допускает JPEG как «перекодированный» вариант; в отчёте явно указано.
- **Broker = Redis Streams** вместо RabbitMQ — обосновано тем, что Redis уже нужен для кэша сессий, AOF включён, это уменьшает количество контейнеров. Если Redis упадёт без AOF-реплики, неопубликованные задачи будут ждать в `attachment_jobs` (outbox) — они не потеряются.
- **DLQ** — пока реестр `attachment_jobs.status='failed'` + `last_error_code`, без отдельного UI. Для оператора — `SELECT` из БД и endpoint replay.
- **Не реализовано:** версионирование нескольких pipeline (трек 5), auto-scaling worker’а по queue lag (трек 1), fault-injection с версионированием (трек 2). Уровень 5 не заявлен.
- **Нагрузка** проверена только smoke + `make test`; полноценных замеров throughput/latency pipeline не делалось (не требуется для уровня 4).
+107
View File
@@ -0,0 +1,107 @@
package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"mtusi-messenger/src/api"
"mtusi-messenger/src/config"
"mtusi-messenger/src/domain"
)
func main() {
cfg := config.Load()
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)")
}
rootCtx, cancelRoot := context.WithCancel(context.Background())
defer cancelRoot()
store, err := domain.NewStore(rootCtx, cfg.DatabaseURL)
if err != nil {
log.Fatalf("postgres: %v", err)
}
defer store.Close()
var s3store *domain.S3Store
if cfg.S3Endpoint != "" {
s3store, err = domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
if err != nil {
log.Fatalf("s3: %v", err)
}
} else {
log.Fatal("S3_ENDPOINT is required for lab4")
}
// Кэш сессий в Redis/Valkey. Если CACHE_URL не задан — работаем без кэша.
var cache *domain.SessionCache
if cfg.CacheURL != "" {
cache, err = domain.NewSessionCache(rootCtx, cfg.CacheURL)
if err != nil {
if cfg.RedisRequired {
log.Fatalf("redis required but unavailable: %v", err)
}
log.Printf("redis unavailable, continuing without session cache: %v", err)
cache = nil
} else {
defer cache.Close()
}
} else if cfg.RedisRequired {
log.Fatal("SESSION_STORE=redis but CACHE_URL is not configured")
}
sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL)
handler := &api.Handler{Store: store, Sessions: sessions, S3: s3store, MaxUpload: cfg.MaxUploadBytes}
router := api.NewRouter(handler, cfg.InstanceID)
go func() {
ticker := time.NewTicker(2 * time.Minute)
defer ticker.Stop()
for {
select {
case <-rootCtx.Done():
return
case <-ticker.C:
if err := store.ReconcileOrphans(rootCtx, s3store, cfg.S3GCGrace); err != nil {
log.Printf("reconcile: %v", err)
}
}
}
}()
srv := &http.Server{
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
Handler: router,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
IdleTimeout: 60 * time.Second,
}
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
go func() {
log.Printf("messenger listening on %s (env=%s, session_ttl=%s)", srv.Addr, cfg.Env, cfg.SessionTTL)
log.Printf("instance_id=%s", cfg.InstanceID)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("server error: %v", err)
}
}()
<-stop
log.Println("shutting down...")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_ = srv.Shutdown(ctx)
log.Println("stopped")
}
+160
View File
@@ -0,0 +1,160 @@
package main
import (
"context"
"errors"
"log"
"os"
"os/signal"
"syscall"
"time"
"mtusi-messenger/src/config"
"mtusi-messenger/src/domain"
)
const (
publishInterval = 1 * time.Second
reapInterval = 30 * time.Second
publishBatch = 50
)
func main() {
cfg := config.Load()
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)")
}
if cfg.CacheURL == "" {
log.Fatal("CACHE_URL is required (env CACHE_URL or CACHE_URL_FILE)")
}
rootCtx, cancelRoot := context.WithCancel(context.Background())
defer cancelRoot()
store, err := domain.NewStore(rootCtx, cfg.DatabaseURL)
if err != nil {
log.Fatalf("postgres: %v", err)
}
defer store.Close()
broker, err := domain.NewBroker(rootCtx, cfg.CacheURL)
if err != nil {
log.Fatalf("broker: %v", err)
}
s3, err := domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
if err != nil {
log.Fatalf("s3: %v", err)
}
go gcLoop(rootCtx, store, s3, cfg.S3GCGrace)
// Reaper зависших processing крутится здесь, потому что outbox уже
// держит соединение с БД и Redis и запускается в единственном экземпляре.
go reaperLoop(rootCtx, store)
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
go func() {
<-stop
log.Println("outbox: shutting down")
cancelRoot()
}()
ticker := time.NewTicker(publishInterval)
defer ticker.Stop()
log.Printf("outbox: running (interval=%s batch=%d)", publishInterval, publishBatch)
for {
select {
case <-rootCtx.Done():
log.Println("outbox: stopped")
return
case <-ticker.C:
if err := publishBatchOnce(rootCtx, store, broker); err != nil {
log.Printf("outbox: publish: %v", err)
}
}
}
}
// publishBatchOnce: выбираем партию неопубликованных job-ов, публикуем
// по одному, после XADD — MarkPublished. Падение между XADD и MarkPublished
// приведёт к повторной публикации при следующем цикле; worker идемпотентен
// (ключ эффекта — (attachment_id, pipeline_version, variant)), поэтому
// дубль в Streams — норма, а не баг.
func publishBatchOnce(ctx context.Context, store *domain.Store, broker *domain.Broker) error {
jobs, err := store.SelectUnpublishedJobs(ctx, publishBatch)
if err != nil {
return err
}
if len(jobs) == 0 {
return nil
}
for _, j := range jobs {
if ctx.Err() != nil {
return ctx.Err()
}
msg := domain.JobMessage{
SchemaVersion: 1,
JobID: j.JobID,
Type: "attachment.process.v1",
AttachmentID: j.AttachmentID,
PipelineVersion: j.PipelineVersion,
Attempt: j.Attempt,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if _, err := broker.Publish(ctx, msg); err != nil {
if errors.Is(err, context.Canceled) {
return err
}
// Не фатально: попробуем в следующем цикле, зафиксировав причину.
_ = store.BumpPublishAttempt(ctx, j.JobID, "publish_failed")
log.Printf("outbox: publish job=%s attachment=%s: %v",
j.JobID, j.AttachmentID, err)
continue
}
if err := store.MarkPublished(ctx, j.JobID); err != nil {
log.Printf("outbox: mark published job=%s: %v", j.JobID, err)
}
}
return nil
}
func reaperLoop(ctx context.Context, store *domain.Store) {
t := time.NewTicker(reapInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
n, err := store.RequeueStuck(ctx)
if err != nil {
log.Printf("outbox: reaper: %v", err)
continue
}
if n > 0 {
log.Printf("outbox: reaper requeued %d stuck job(s)", n)
}
}
}
}
func gcLoop(ctx context.Context, store *domain.Store, s3 *domain.S3Store, grace time.Duration) {
t := time.NewTicker(2 * time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := store.ReconcileOrphans(ctx, s3, grace); err != nil {
log.Printf("outbox: gc: %v", err)
}
}
}
}
+213
View File
@@ -0,0 +1,213 @@
package main
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"os/signal"
"syscall"
"time"
"mtusi-messenger/src/config"
"mtusi-messenger/src/domain"
"github.com/redis/go-redis/v9"
)
const (
lease = 60 * time.Second
maxAttempts = 5
baseBackoff = 2 * time.Second
)
func main() {
cfg := config.Load()
if cfg.CacheURL == "" {
log.Fatal("CACHE_URL (redis) is required for broker")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := domain.NewStore(ctx, cfg.DatabaseURL)
if err != nil {
log.Fatalf("db: %v", err)
}
defer store.Close()
s3, err := domain.NewS3Store(ctx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
if err != nil {
log.Fatalf("s3: %v", err)
}
broker, err := domain.NewBroker(ctx, cfg.CacheURL)
if err != nil {
log.Fatalf("broker: %v", err)
}
consumer := cfg.InstanceID + "-worker"
go reaperLoop(ctx, store)
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
go func() {
<-stop
log.Println("worker shutting down")
cancel()
}()
for ctx.Err() == nil {
msgs, err := broker.Consume(ctx, consumer)
if err != nil {
if ctx.Err() != nil {
return
}
log.Printf("consume: %v", err)
time.Sleep(time.Second)
continue
}
for _, m := range msgs {
handleOne(ctx, store, s3, broker, m)
}
}
}
func reaperLoop(ctx context.Context, store *domain.Store) {
t := time.NewTicker(30 * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if n, err := store.RequeueStuck(ctx); err != nil {
log.Printf("reaper: %v", err)
} else if n > 0 {
log.Printf("reaper: requeued %d stuck job(s)", n)
}
}
}
}
func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store,
broker *domain.Broker, m redis.XMessage) {
var job domain.JobMessage
if raw, ok := m.Values["payload"].(string); ok {
_ = json.Unmarshal([]byte(raw), &job)
} else if raw, ok := m.Values["payload"].([]byte); ok {
_ = json.Unmarshal(raw, &job)
} else {
_ = broker.Ack(ctx, m.ID)
return
}
// Валидация схемы (обязательные поля).
if job.SchemaVersion != 1 || job.Type != "attachment.process.v1" ||
job.JobID == "" || job.AttachmentID == "" {
log.Printf("poison message %s, acking", m.ID)
_ = broker.Ack(ctx, m.ID)
return
}
// No-op для уже готового результата (redelivery после готовности).
if done, _ := store.VariantsExist(ctx, job.AttachmentID, job.PipelineVersion); done {
_ = broker.Ack(ctx, m.ID)
return
}
j, err := store.ClaimJobByID(ctx, job.JobID, workerID(), lease)
if errors.Is(err, domain.ErrNotFound) {
// Уже done/failed или lease держит кто-то другой — ack, не дублируем.
_ = broker.Ack(ctx, m.ID)
return
}
if err != nil {
log.Printf("claim: %v", err)
return
}
stagingKey, _, err := store.GetStagingKey(ctx, j.AttachmentID)
if err != nil {
log.Printf("staging key: %v", err)
return
}
raw, ct, err := s3.FetchStagingForProcessing(ctx, stagingKey)
if err != nil {
code, permanent := classify(err)
if permanent {
// постоянная ошибка — failed без retry
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, code)
} else {
_, _ = store.RetryJob(ctx, j.JobID, code, maxAttempts, baseBackoff)
}
_ = broker.Ack(ctx, m.ID)
return
}
variants, err := processImage(raw, ct)
if err != nil {
// decode error — постоянная
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, "invalid_image")
_ = broker.Ack(ctx, m.ID)
return
}
finalOriginalKey := "final/" + j.AttachmentID + "/original"
var vrows []domain.Variant
for _, v := range variants {
if v.Name == "original" {
// оригинал перекладываем из staging server-side.
if err := s3.MoveToFinal(ctx, stagingKey, finalOriginalKey); err != nil {
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
_ = broker.Ack(ctx, m.ID)
return
}
} else {
key := "final/" + j.AttachmentID + "/" + v.Name
sum, err := s3.PutBytes(ctx, key, v.ContentType, v.Data)
if err != nil {
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
_ = broker.Ack(ctx, m.ID)
return
}
v.SHA256 = sum
}
vrows = append(vrows, domain.Variant{
Name: v.Name, ContentType: v.ContentType, SizeBytes: int64(len(v.Data)),
Width: v.Width, Height: v.Height, SHA256: v.SHA256,
})
}
if err := store.MarkReady(ctx, j.JobID, j.AttachmentID, j.PipelineVersion, finalOriginalKey, vrows); err != nil {
// Уже done — не страшно; иначе retry.
if !errors.Is(err, domain.ErrConflict) {
log.Printf("mark ready: %v", err)
}
_ = broker.Ack(ctx, m.ID)
return
}
_ = broker.Ack(ctx, m.ID)
}
func workerID() string {
h, _ := os.Hostname()
return h + "-" + time.Now().Format("150405")
}
func classify(err error) (string, bool) {
switch {
case errors.Is(err, domain.ErrTooLarge):
return "payload_too_large", true
case errors.Is(err, domain.ErrUnsupported):
return "unsupported_media_type", true
case errors.Is(err, domain.ErrNotImage):
return "invalid_image", true
default:
return "storage_error", false
}
}
+137
View File
@@ -0,0 +1,137 @@
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"image"
"image/jpeg"
"math"
"golang.org/x/image/draw"
)
type VariantSpec struct {
Name string
ContentType string
Width int
Height int
Data []byte
SHA256 string
}
func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, src, &jpeg.Options{Quality: quality}); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// processImage возвращает 4 варианта. Оригинал — byte-identical с входом.
// thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop),
// optimized — JPEG q=70 без метаданных (WebP оставлен на усмотрение; см. отчёт).
func processImage(raw []byte, contentType string) ([]VariantSpec, error) {
src, _, err := image.Decode(bytes.NewReader(raw))
if err != nil {
return nil, err
}
out := make([]VariantSpec, 0, 4)
// original — ровно те байты, что пришли в staging.
sum := sha256.Sum256(raw)
out = append(out, VariantSpec{
Name: "original", ContentType: contentType,
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
Data: raw, SHA256: hex.EncodeToString(sum[:]),
})
// thumbnail: fit в 256x160 с сохранением пропорций (для 16:10 — 256x160).
th := resizeFit(src, 256, 256)
if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil {
return nil, err
}
// crop 128x128: центр-cover до меньшего квадрата.
cr := squareCover(src, 128)
if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil {
return nil, err
}
// optimized
optBuf, err := encodeOptimizedJPEG(src, 70)
if err != nil {
return nil, err
}
optSum := sha256.Sum256(optBuf)
out = append(out, VariantSpec{
Name: "optimized", ContentType: "image/jpeg",
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
Data: optBuf, SHA256: hex.EncodeToString(optSum[:]),
})
return out, nil
}
func appendEncoded(out *[]VariantSpec, name, ct string, img image.Image) error {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil {
return err
}
sum := sha256.Sum256(buf.Bytes())
*out = append(*out, VariantSpec{
Name: name, ContentType: ct,
Width: img.Bounds().Dx(), Height: img.Bounds().Dy(),
Data: buf.Bytes(), SHA256: hex.EncodeToString(sum[:]),
})
return nil
}
func resizeFit(src image.Image, maxW, maxH int) image.Image {
b := src.Bounds()
w, h := b.Dx(), b.Dy()
scale := float64(maxW) / float64(w)
if float64(maxH)/float64(h) < scale {
scale = float64(maxH) / float64(h)
}
if scale > 1 {
scale = 1
}
nw := int(math.Round(float64(w) * scale))
nh := int(math.Round(float64(h) * scale))
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil)
return dst
}
// squareCover масштабирует src так, чтобы покрыть квадрат size×size
// (min-масштаб по двум осям), затем центр-кроп до size×size.
// Результат — всегда ровно size×size.
func squareCover(src image.Image, size int) image.Image {
b := src.Bounds()
w, h := b.Dx(), b.Dy()
// scale — больший из двух, чтобы покрыть квадрат
scale := float64(size) / float64(w)
if float64(size)/float64(h) > scale {
scale = float64(size) / float64(h)
}
nw := int(math.Round(float64(w) * scale))
nh := int(math.Round(float64(h) * scale))
if nw < size {
nw = size
}
if nh < size {
nh = size
}
scaled := image.NewRGBA(image.Rect(0, 0, nw, nh))
draw.CatmullRom.Scale(scaled, scaled.Bounds(), src, b, draw.Over, nil)
// center-crop до size×size
x0 := (nw - size) / 2
y0 := (nh - size) / 2
dst := image.NewRGBA(image.Rect(0, 0, size, size))
draw.Draw(dst, dst.Bounds(), scaled,
image.Pt(x0, y0), draw.Src)
return dst
}
+162
View File
@@ -0,0 +1,162 @@
x-api: &api
build:
context: .
dockerfile: Dockerfile
env_file:
- .env
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
s3: { condition: service_healthy }
s3-init: { condition: service_completed_successfully }
migrate: { condition: service_completed_successfully }
networks: [backend]
services:
postgres:
image: postgres:16.3-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 20
networks: [backend]
deploy:
resources:
limits: { cpus: "${PG_CPUS:-1.0}", memory: "${PG_MEM:-512M}" }
redis:
image: redis:7.2.5-alpine
command: ["redis-server", "--appendonly", "yes", "--save"]
volumes: [redisdata:/data]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
networks: [backend]
s3:
image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
secrets: [minio_root_user, minio_root_password]
volumes: [s3data:/data]
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 3s
retries: 20
networks: [backend]
deploy:
resources:
limits: { cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEM:-512M}" }
s3-init:
image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z
depends_on:
s3: { condition: service_healthy }
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
entrypoint: ["/bin/sh", "/bootstrap/bootstrap.sh"]
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro
- /tmp/silo-init-data:/data
networks: [backend]
deploy:
replicas: 1
restart_policy: { condition: "no" }
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_PER_REPLICA:-0.5}"
memory: "${API_MEM_PER_REPLICA:-128M}"
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
interval: 10s
timeout: 3s
start_period: 20s
retries: 5
outbox:
<<: *api
entrypoint: ["/app/outbox"]
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
deploy:
replicas: 1
resources:
limits: { cpus: "0.2", memory: "64M" }
worker:
<<: *api
entrypoint: ["/app/worker"]
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
s3: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
deploy:
replicas: 2
resources:
limits: { cpus: "0.5", memory: "256M" }
migrate:
<<: *api
entrypoint: ["/app/migrate.sh"]
depends_on:
postgres: { condition: service_healthy }
deploy:
replicas: 1
restart_policy:
condition: "no"
lb:
image: nginx:1.27.0-alpine
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "8080:8080"
depends_on: [api]
networks: [backend]
volumes:
pgdata:
redisdata:
s3data:
networks:
backend:
driver: bridge
secrets:
postgres_password: { file: ./secrets/postgres_password.txt }
database_url: { file: ./secrets/database_url.txt }
cache_url: { file: ./secrets/cache_url.txt }
s3_access_key_id: { file: ./secrets/s3_access_key_id.txt }
s3_secret_access_key: { file: ./secrets/s3_secret_access_key.txt }
minio_root_user: { file: ./secrets/minio_root_user.txt }
minio_root_password: { file: ./secrets/minio_root_password.txt }
Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 94 KiB

+56
View File
@@ -0,0 +1,56 @@
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ make my_tests
python3 student_tests/states.py
upload -> queued
status stays queued while worker paused
ready with thumbnail/crop/optimized (original available via ?variant=original)
original served via ?variant=original (111817 bytes)
python3 student_tests/limits.py
413 payload_too_large
415 unsupported_media_type
415 mismatched content (client MIME not trusted)
400 invalid image (signature ok, header broken)
400 too many pixels
python3 student_tests/variants.py
original byte-identical
thumbnail 256x160 (metadata matches)
crop 128x128 JPEG
optimized present (4818 bytes)
python3 student_tests/broker-payload.py
stream sample:
1794848167433-0
payload
{"schema_version":1,"job_id":"ee59551f-c85f-411a-b4d9-6076f31fde8f","type":"attachment.process.v1","attachment_id":"1175cf24-2984-4800-b84d-1aa702ecd7bd","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:07Z"}
1794848176445-0
payload
{"schema_version":1,"job_id":"b7792daf-6123-4c7a-9566-b2fc4a53a614","type":"attachment.process.v1","attachment_id":"05ef10e7-19c4-47ef-9c68-a2e09eb8338e","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:16Z"}
broker payload has only ids/version/attempt
python3 student_tests/staging-cleanup.py
staging object removed after ready (mc rc=1)
python3 student_tests/dlq.py
failed registry:
b1ccbeca-ae22-44f1-837c-868d2165889c|invalid_image|1
python3 student_tests/redelivery.py
redelivery is no-op (still 3 non-original variants; original via object_key)
python3 student_tests/outbox.py
outbox paused: published_at IS NULL, upload accepted
after unpause: ready
python3 student_tests/failed.py
failed immediately (attempt=1), error_code=invalid_image — permanent, not retried
python3 student_tests/retry.py
transient error retried: attempt=2
retry log lines:
worker-1 | 2026/11/16 16:56:48 retry job=64072f3d-6b9e-4101-b31b-7e9a2f51bac3 attempt=2 next_attempt=2026-11-16T16:56:50.964432673Z backoff=2s jitter=451.35473ms code=storage_error
recovered to ready after staging restored (attempt=2)
python3 student_tests/replay.py
failed reached
replay -> queued
replay reset attempt to 1; permanent error failed again — not retried
python3 student_tests/lease.py
worker stopped at processing (job=da0adcf7-64a7-427c-a50b-438e9d688bb7, lease_expires_at=2026-11-16 16:57:59.829609+00)
reaper requeued the stuck job
ready after unstuck
python3 student_tests/crash-before-ack.py
worker killed during processing
recovery produced exactly 3 non-original variants, no duplicates (original in object_key)
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$
+31
View File
@@ -0,0 +1,31 @@
module mtusi-messenger
go 1.26.0
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.5.5
github.com/minio/minio-go/v7 v7.0.72
github.com/redis/go-redis/v9 v9.5.1
golang.org/x/crypto v0.23.0
golang.org/x/image v0.48.0
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/rs/xid v1.5.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/sync v0.25.0 // indirect
golang.org/x/sys v0.50.0 // indirect
golang.org/x/text v0.44.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
+64
View File
@@ -0,0 +1,64 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA=
github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+52
View File
@@ -0,0 +1,52 @@
#!/bin/sh
set -eu
MINIO_ROOT_USER_FILE="${MINIO_ROOT_USER_FILE:-/run/secrets/minio_root_user}"
MINIO_ROOT_PASSWORD_FILE="${MINIO_ROOT_PASSWORD_FILE:-/run/secrets/minio_root_password}"
S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}"
S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
API_USER_FILE="${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}"
API_PASS_FILE="${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}"
for f in "$MINIO_ROOT_USER_FILE" "$MINIO_ROOT_PASSWORD_FILE"; do
[ -f "$f" ] || { echo "ERROR: missing $f" >&2; exit 1; }
done
ROOT_USER="$(cat "$MINIO_ROOT_USER_FILE")"
ROOT_PASS="$(cat "$MINIO_ROOT_PASSWORD_FILE")"
mc alias set local "$S3_ENDPOINT" "$ROOT_USER" "$ROOT_PASS" >/dev/null
# 1) bucket: приватный, создаётся один раз
mc mb --ignore-existing "local/$S3_BUCKET" >/dev/null
mc anonymous set none "local/$S3_BUCKET" >/dev/null
# 2) ограниченный пользователь API (если креды заданы заранее)
if [ -s "$API_USER_FILE" ] && [ -s "$API_PASS_FILE" ]; then
API_USER="$(cat "$API_USER_FILE")"
API_PASS="$(cat "$API_PASS_FILE")"
if ! mc admin user info local "$API_USER" >/dev/null 2>&1; then
mc admin user add local "$API_USER" "$API_PASS" >/dev/null
fi
# Политика: только PutObject/GetObject/DeleteObject в bucket.
cat >/tmp/api-policy.json <<EOF
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}]
}
EOF
mc admin policy create local api-attachments /tmp/api-policy.json >/dev/null 2>&1 || true
mc admin policy attach local api-attachments --user "$API_USER" >/dev/null 2>&1 || true
else
echo "WARN: API S3 credentials not pre-created; generate and place them in secrets/" >&2
fi
echo "s3 bootstrap done (bucket=$S3_BUCKET)"
+29
View File
@@ -0,0 +1,29 @@
worker_processes auto;
events { worker_connections 4096; }
http {
# Docker's embedded DNS. nginx будет обращаться сюда за IP реплик.
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 5s;
server {
listen 8080;
client_max_body_size 12m;
location / {
set $api_backend http://api:8080;
proxy_pass $api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_connect_timeout 2s;
proxy_send_timeout 15s;
proxy_read_timeout 15s;
proxy_next_upstream error timeout http_502 http_503 http_504;
}
}
}
+58
View File
@@ -0,0 +1,58 @@
-- ============================================================
-- Core: users, sessions, chats, members, messages, idempotency
-- ============================================================
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
status_code INTEGER NOT NULL,
response JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_keys (created_at);
+88
View File
@@ -0,0 +1,88 @@
-- ============================================================
-- Attachments: metadata, variants, jobs/outbox, GC
-- ============================================================
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
object_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0 AND size_bytes <= 10485760),
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
width INTEGER NOT NULL CHECK (width >= 0),
height INTEGER NOT NULL CHECK (height >= 0),
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','processing','ready','failed','pending')),
error_code TEXT,
-- поля из 0006
pipeline_version INTEGER NOT NULL DEFAULT 1,
staging_key TEXT,
ready_at TIMESTAMPTZ,
failed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id);
-- Для reconciler/поиска «зависших» pending-записей:
CREATE INDEX IF NOT EXISTS idx_attachments_pending
ON attachments (created_at) WHERE status <> 'ready';
-- Связь сообщение ↔ вложения. FK на элементы массива Postgres не умеет,
-- целостность — на уровне приложения (или триггера, если нужно).
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
-- Журнал компенсации: объекты S3, которые могли остаться без метаданных.
CREATE TABLE IF NOT EXISTS attachment_gc (
object_key TEXT PRIMARY KEY,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Idempotency эффекта: (attachment_id, pipeline_version, variant)
-- content_type без CHECK: варианты могут быть webp и т.п.
-- sha256 — text+CHECK, чтобы не ловить padding от char(64).
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
variant TEXT NOT NULL,
object_key TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
-- Outbox + очередь job-ов
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id UUID PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','processing','done','failed')),
-- outbox
published_at TIMESTAMPTZ,
publish_attempts INTEGER NOT NULL DEFAULT 0,
-- lease
lease_owner TEXT,
lease_expires_at TIMESTAMPTZ,
-- retry
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию
);
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished
ON attachment_jobs (next_attempt_at)
WHERE published_at IS NULL AND status = 'queued';
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
ON attachment_jobs (lease_expires_at)
WHERE status = 'processing';
@@ -0,0 +1,5 @@
-- Индекс B для «стоимости записи»:
-- покрывающий индекс, включающий text — растёт быстрее, помогает только
-- покрывающим запросам. Накатывать отдельно, чтобы сравнить с индексом A.
CREATE INDEX IF NOT EXISTS idx_messages_chat_covering
ON messages (chat_id, created_at, id) INCLUDE (sender_id, text);
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
# Применяет миграции к БД, читая DATABASE_URL из файла секрета.
# Устойчиво к повторному запуску (CREATE ... IF NOT EXISTS).
URL_FILE="${DATABASE_URL_FILE:-/run/secrets/database_url}"
DATABASE_URL="$(cat "$URL_FILE")"
PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1 -q"
echo "waiting for postgres..."
for i in $(seq 1 60); do
if psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null 2>&1; then
echo "postgres is ready"
break
fi
echo " attempt $i: not ready yet"
sleep 1
done
psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null
for f in /app/migrations/*.sql; do
echo "applying $f"
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$f"
done
echo "migrations done"
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
# ./scripts/restart.sh # 2 реплики (по умолчанию)
# ./scripts/restart.sh 4 # 4 реплики
REPLICAS="${1:-2}"
# Останов с сохранением volumes.
docker compose down --remove-orphans
# Пересоздание сети + контейнеров с нужным числом реплик.
API_REPLICAS="$REPLICAS" \
docker compose --env-file .env -f compose.yaml up -d --build
# Ждём готовности с хоста.
i=0
while [ "$i" -lt 90 ]; do
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s (replicas=$REPLICAS)"
exit 0
fi
i=$((i + 1))
sleep 1
done
echo "stack did not become ready in 90s" >&2
exit 1
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
set -eu
# Все контейнеры сервиса api (включая Exited).
# `docker compose ps -a -q api` возвращает их ID.
STOPPED=""
for id in $(docker compose ps -a -q api); do
state=$(docker inspect --format '{{.State.Status}}' "$id")
if [ "$state" = "exited" ] || [ "$state" = "created" ]; then
STOPPED="$STOPPED $id"
fi
done
if [ -z "$STOPPED" ]; then
echo "all api containers already running"
exit 0
fi
for id in $STOPPED; do
hn=$(docker inspect --format '{{.Config.Hostname}}' "$id")
docker start "$id" >/dev/null
echo "started $id ($hn)"
done
# Дождаться, пока LB увидит их снова (curl с хоста).
i=0
while [ "$i" -lt 30 ]; do
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s"
exit 0
fi
i=$((i + 1))
sleep 1
done
echo "stack did not become ready in 30s" >&2
exit 1
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
# Находим ID контейнера, чьё имя (HOSTNAME) совпадает с TARGET_INSTANCE_ID.
CID=$(docker compose ps -q api | while read -r id; do
name=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$name" = "$TARGET_INSTANCE_ID" ]; then
echo "$id"
break
fi
done)
if [ -z "$CID" ]; then
echo "container with hostname $TARGET_INSTANCE_ID not found" >&2
exit 2
fi
docker stop "$CID"
echo "stopped $CID ($TARGET_INSTANCE_ID)"
+79
View File
@@ -0,0 +1,79 @@
package api
import (
"context"
"encoding/base64"
"errors"
"net/http"
"strings"
"mtusi-messenger/src/domain"
)
// bearerAuth извлекает токен, проверяет сессию и кладёт пользователя в контекст.
// Basic на бизнес-эндпоинтах не принимается. X-User-Id игнорируется.
func (h *Handler) bearerAuth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
header := r.Header.Get("Authorization")
if !strings.HasPrefix(header, "Bearer ") {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token")
return
}
token := strings.TrimSpace(strings.TrimPrefix(header, "Bearer "))
if token == "" {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
writeError(w, r, http.StatusUnauthorized, "unauthorized", "missing bearer token")
return
}
u, err := h.Sessions.Lookup(r.Context(), token)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrUnauthorized):
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid session")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
return
}
ctx := context.WithValue(r.Context(), userKey, *u)
ctx = context.WithValue(ctx, tokenKey, token)
next(w, r.WithContext(ctx))
}
}
func userFrom(r *http.Request) (domain.User, bool) {
u, ok := r.Context().Value(userKey).(domain.User)
return u, ok
}
func tokenFrom(r *http.Request) string {
if v, ok := r.Context().Value(tokenKey).(string); ok {
return v
}
return ""
}
// parseBasicCredentials разбирает Authorization: Basic base64(username:password).
func parseBasicCredentials(header string) (string, string, error) {
if !strings.HasPrefix(header, "Basic ") {
return "", "", errors.New("not basic")
}
raw := strings.TrimSpace(strings.TrimPrefix(header, "Basic "))
decoded, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
return "", "", errors.New("invalid base64")
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
return "", "", errors.New("missing colon")
}
return parts[0], parts[1], nil
}
+28
View File
@@ -0,0 +1,28 @@
package api
import (
"encoding/json"
"net/http"
"mtusi-messenger/src/domain"
)
// writeError пишет ошибку в формате контракта.
func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(domain.ErrorBody{
Error: domain.ErrorDetail{
Code: code,
Message: message,
RequestID: requestIDFrom(r),
},
})
}
// writeJSON пишет успешный JSON-ответ.
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
+603
View File
@@ -0,0 +1,603 @@
package api
import (
"bytes"
"encoding/json"
"errors"
"io"
"log"
"net/http"
"regexp"
"strconv"
"strings"
"mtusi-messenger/src/domain"
"github.com/google/uuid"
)
var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
type Handler struct {
Store *domain.Store
Sessions *domain.SessionStore
S3 *domain.S3Store
MaxUpload int64
}
// --- Health ---
func (h *Handler) Live(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
// Ready проверяет БД (обязательная зависимость). Redis — кэш, его недоступность
// не должна ронять readiness: API может работать, обходя кэш.
func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) {
if err := h.Store.Ping(r.Context()); err != nil {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
// --- Users ---
type createUserReq struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) {
var req createUserReq
if err := decodeSingleJSON(r.Body, &req); err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
return
}
if !usernameRe.MatchString(req.Username) {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid username")
return
}
if n := len([]rune(req.DisplayName)); n < 1 || n > 100 || strings.TrimSpace(req.DisplayName) == "" {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid display_name")
return
}
// Пароль: 12–128 code points, не состоит только из пробелов.
if n := len([]rune(req.Password)); n < 12 || n > 128 || strings.TrimSpace(req.Password) == "" {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid password")
return
}
u, err := h.Store.CreateUser(r.Context(), req.Username, req.DisplayName, req.Password)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrConflict):
writeError(w, r, http.StatusConflict, "conflict", "username taken")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusCreated, u)
}
func (h *Handler) CurrentUser(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
u, _ := userFrom(r)
writeJSON(w, http.StatusOK, u)
})(w, r)
}
// --- Auth ---
func (h *Handler) CreateSession(w http.ResponseWriter, r *http.Request) {
username, password, err := parseBasicCredentials(r.Header.Get("Authorization"))
if err != nil {
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials")
return
}
u, err := h.Store.VerifyPassword(r.Context(), username, password)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrUnauthorized):
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
writeError(w, r, http.StatusUnauthorized, "unauthorized", "invalid credentials")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
s, err := h.Sessions.Create(r.Context(), *u)
if err != nil {
if errors.Is(err, domain.ErrTimeout) {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
return
}
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
return
}
writeJSON(w, http.StatusCreated, s)
}
func (h *Handler) RevokeSession(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
token := tokenFrom(r)
if err := h.Sessions.Revoke(r.Context(), token); err != nil {
if errors.Is(err, domain.ErrTimeout) {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store timeout")
return
}
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "session store unavailable")
return
}
w.WriteHeader(http.StatusNoContent)
})(w, r)
}
// --- Chats ---
type createChatReq struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
func (h *Handler) CreateChat(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
var req createChatReq
if err := decodeSingleJSON(r.Body, &req); err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
return
}
if strings.TrimSpace(req.Title) == "" || len([]rune(req.Title)) > 100 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid title")
return
}
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid member_ids")
return
}
c, err := h.Store.CreateChat(r.Context(), req.Title, req.MemberIDs, me.ID)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid members")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusCreated, c)
})(w, r)
}
func (h *Handler) ListChats(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
limit := 50
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 100 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit")
return
}
limit = n
}
cursor := r.URL.Query().Get("cursor")
chats, next, err := h.Store.ListChats(r.Context(), me.ID, limit, cursor)
switch {
case err == nil:
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusOK, domain.ChatPage{Items: chats, NextCursor: next})
})(w, r)
}
func (h *Handler) GetChat(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
chatID := r.PathValue("chat_id")
c, err := h.Store.GetChat(r.Context(), chatID, me.ID)
switch {
case err == nil:
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
return
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusOK, c)
})(w, r)
}
// --- Messages ---
type createMessageReq struct {
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}
func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
chatID := r.PathValue("chat_id")
// Idempotency-Key обязателен.
key := r.Header.Get("Idempotency-Key")
if key == "" {
writeError(w, r, http.StatusBadRequest, "invalid_request", "Idempotency-Key required")
return
}
if !idempotencyKeyRe.MatchString(key) {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid Idempotency-Key")
return
}
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
switch {
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
return
}
// Читаем тело один раз и хешируем нормализованный JSON.
rawBody, err := io.ReadAll(io.LimitReader(r.Body, 64*1024))
if err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "cannot read body")
return
}
var req createMessageReq
if err := decodeSingleJSON(bytes.NewReader(rawBody), &req); err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
return
}
if len([]rune(req.Text)) > 4000 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "message too long")
return
}
if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message")
return
}
bodyHash, err := domain.HashBody(rawBody)
if err != nil {
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
return
}
if len(req.Text) > 0 && len([]rune(req.Text)) == 0 {
// всё пробелы, но вложения есть — допустимо
}
attachmentIDs := req.AttachmentIDs
if attachmentIDs == nil {
attachmentIDs = []string{}
}
msg, rec, replay, err := h.Store.AddMessageIdempotent(
r.Context(), chatID, me.ID, req.Text, attachmentIDs, key, bodyHash)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
return
case errors.Is(err, domain.ErrConflict):
writeError(w, r, http.StatusConflict, "conflict", "idempotency key reused with different body")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
if replay {
// Возвращаем сохранённый 201 и то же Message.
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(rec.StatusCode)
_, _ = w.Write(rec.Response)
return
}
writeJSON(w, http.StatusCreated, msg)
})(w, r)
}
func (h *Handler) ListMessages(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
chatID := r.PathValue("chat_id")
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
switch {
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
return
}
limit := 50
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 100 {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid limit")
return
}
limit = n
}
cursor := r.URL.Query().Get("cursor")
items, next, err := h.Store.ListMessages(r.Context(), chatID, limit, cursor)
switch {
case err == nil:
// ok
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid cursor")
return
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "query timeout")
return
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusOK, domain.MessagePage{Items: items, NextCursor: next})
})(w, r)
}
func decodeSingleJSON(r io.Reader, v any) error {
dec := json.NewDecoder(r)
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
return err
}
if dec.More() {
return errors.New("trailing data after JSON value")
}
// Пробуем прочитать ещё один токен — если это не EOF, значит есть мусор.
var extra any
if err := dec.Decode(&extra); err != io.EOF {
return errors.New("trailing data after JSON value")
}
return nil
}
// --- Attachments ---
// safeFilename отбрасывает пути и опасные имена; используется только для отображения.
func safeFilename(name string) (string, bool) {
name = strings.TrimSpace(name)
if name == "" || len(name) > 255 {
return "", false
}
if strings.ContainsAny(name, "/\\") || strings.Contains(name, "..") {
return "", false
}
for _, r := range name {
if r < 0x20 || r == 0x7f {
return "", false
}
}
return name, true
}
func errorCodeFor(err error) string {
switch {
case errors.Is(err, domain.ErrTooLarge):
return "payload_too_large"
case errors.Is(err, domain.ErrUnsupported):
return "unsupported_media_type"
case errors.Is(err, domain.ErrNotImage):
return "invalid_image"
default:
return "storage_error"
}
}
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
chatID := r.PathValue("chat_id")
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
switch {
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
return
}
filename, ok := safeFilename(r.URL.Query().Get("filename"))
if !ok {
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid filename")
return
}
stagingKey := domain.StagingPrefix + chatID + "/" + uuid.NewString()
pendingID, err := h.Store.CreateAttachmentPending(
r.Context(), chatID, me.ID, stagingKey, filename)
if err != nil {
switch {
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid attachment metadata")
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
return
}
res, err := h.S3.PutValidatedImage(r.Context(), stagingKey, r.Body)
if err != nil {
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, errorCodeFor(err))
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "upload_failed")
switch {
case errors.Is(err, domain.ErrTooLarge):
writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large")
case errors.Is(err, domain.ErrUnsupported):
writeError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "only PNG/JPEG")
case errors.Is(err, domain.ErrNotImage):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid image")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
}
return
}
// Атомарно: attachment queued + outbox-строка. 202 только после COMMIT.
att, err := h.Store.EnqueueAttachment(
r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256,
res.Width, res.Height, domain.PipelineVersion)
if err != nil {
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "enqueue_failed")
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusAccepted, att)
})(w, r)
}
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
id := r.PathValue("attachment_id")
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
switch {
case err == nil:
writeJSON(w, http.StatusOK, att)
case errors.Is(err, domain.ErrNotFound):
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
})(w, r)
}
func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
id := r.PathValue("attachment_id")
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
return
}
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
switch att.Status {
case domain.AttachmentReady:
// ok
case domain.AttachmentFailed:
writeError(w, r, http.StatusConflict, "conflict", "attachment failed")
return
default:
writeError(w, r, http.StatusConflict, "conflict", "attachment not ready")
return
}
variant := r.URL.Query().Get("variant")
if variant == "" {
variant = "original"
}
var key string
if variant == "original" {
key, err = h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
} else {
key, _, err = h.Store.GetVariantKey(r.Context(), att.ID, att.PipelineVersion, variant)
}
if err != nil {
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
return
}
rc, contentType, size, err := h.S3.GetObject(r.Context(), key)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
writeError(w, r, http.StatusNotFound, "not_found", "file missing in storage")
return
}
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
return
}
defer rc.Close()
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.Header().Set("Content-Disposition", `inline; filename="`+strings.ReplaceAll(att.Filename, `"`, "")+`"`)
w.WriteHeader(http.StatusOK)
written, err := io.Copy(w, rc)
if err != nil || written != size {
// Соединение уже частично отправлено, но клиент получит обрезанный файл.
// Логируем это для отладки.
log.Printf("download truncated: written=%d expected=%d err=%v", written, size, err)
}
})(w, r)
}
func (h *Handler) ReplayAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
// Только owner или участник чата; упрощённо — участник чата.
me, _ := userFrom(r)
id := r.PathValue("attachment_id")
if _, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID); err != nil {
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
return
}
err := h.Store.ReplayFailed(r.Context(), id)
if errors.Is(err, domain.ErrConflict) {
writeError(w, r, http.StatusConflict, "conflict", "not in failed state")
return
}
if err != nil {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
w.WriteHeader(http.StatusAccepted)
})(w, r)
}
+46
View File
@@ -0,0 +1,46 @@
package api
import (
"context"
"net/http"
"github.com/google/uuid"
)
type ctxKey string
const (
requestIDKey ctxKey = "request_id"
userKey ctxKey = "user"
tokenKey ctxKey = "token"
)
func requestIDFrom(r *http.Request) string {
if v, ok := r.Context().Value(requestIDKey).(string); ok {
return v
}
return ""
}
func withRequestID(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := r.Header.Get("X-Request-Id")
if id == "" {
id = uuid.NewString()
}
w.Header().Set("X-Request-Id", id)
ctx := context.WithValue(r.Context(), requestIDKey, id)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
// withInstanceID ставит X-Instance-Id на КАЖДЫЙ ответ (включая ошибки и health).
// ID генерируется один раз при старте процесса — стабилен в пределах жизни.
func withInstanceID(instanceID string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
next.ServeHTTP(w, r)
})
}
}
+25
View File
@@ -0,0 +1,25 @@
package api
import "net/http"
func NewRouter(h *Handler, instanceID string) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /health/live", h.Live)
mux.HandleFunc("GET /health/ready", h.Ready)
mux.HandleFunc("POST /api/v1/users", h.CreateUser)
mux.HandleFunc("GET /api/v1/users/me", h.CurrentUser)
mux.HandleFunc("POST /api/v1/auth/sessions", h.CreateSession)
mux.HandleFunc("DELETE /api/v1/auth/sessions/current", h.RevokeSession)
mux.HandleFunc("POST /api/v1/chats", h.CreateChat)
mux.HandleFunc("GET /api/v1/chats", h.ListChats)
mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat)
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage)
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages)
mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment)
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment)
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment)
mux.HandleFunc("POST /api/v1/attachments/{attachment_id}/replay", h.ReplayAttachment)
// Порядок: instanceID снаружи, requestID внутри.
return withInstanceID(instanceID)(withRequestID(mux))
}
+136
View File
@@ -0,0 +1,136 @@
package config
import (
"crypto/rand"
"encoding/hex"
"os"
"strconv"
"time"
)
type Config struct {
Host string
Port int
Env string
DatabaseURL string
CacheURL string
SessionTTL time.Duration
SessionStore string
RedisRequired bool
InstanceID string
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKeyID string
S3SecretKey string
MaxUploadBytes int64
MaxImagePixels int64
S3GCGrace time.Duration
}
func getenv(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
func readSecretFromFile(path string) string {
if path == "" {
return ""
}
b, err := os.ReadFile(path)
if err != nil {
return ""
}
return string(b)
}
func Load() Config {
port := 8080
if v := os.Getenv("APP_PORT"); v != "" {
if p, err := strconv.Atoi(v); err == nil {
port = p
}
}
ttl := 3600
if v := os.Getenv("SESSION_TTL_SECONDS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
ttl = n
}
}
// DATABASE_URL и CACHE_URL могут прийти либо напрямую, либо через *_FILE.
dbURL := os.Getenv("DATABASE_URL")
if dbURL == "" {
dbURL = readSecretFromFile(os.Getenv("DATABASE_URL_FILE"))
}
cacheURL := os.Getenv("CACHE_URL")
if cacheURL == "" {
cacheURL = readSecretFromFile(os.Getenv("CACHE_URL_FILE"))
}
instanceID := os.Getenv("INSTANCE_ID")
if instanceID == "" {
instanceID = os.Getenv("HOSTNAME")
}
if instanceID == "" {
instanceID = "api-" + randomShortID()
}
store := getenv("SESSION_STORE", "postgres")
s3Access := os.Getenv("S3_ACCESS_KEY_ID")
if s3Access == "" {
s3Access = readSecretFromFile(os.Getenv("S3_ACCESS_KEY_ID_FILE"))
}
s3Secret := os.Getenv("S3_SECRET_ACCESS_KEY")
if s3Secret == "" {
s3Secret = readSecretFromFile(os.Getenv("S3_SECRET_ACCESS_KEY_FILE"))
}
maxUpload := int64(10485760)
if v := os.Getenv("MAX_UPLOAD_BYTES"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
maxUpload = n
}
}
maxPixels := int64(40000000)
if v := os.Getenv("MAX_IMAGE_PIXELS"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
maxPixels = n
}
}
gcGrace := 600 * time.Second
if v := os.Getenv("S3_GC_GRACE_SECONDS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
gcGrace = time.Duration(n) * time.Second
}
}
return Config{
Host: getenv("APP_HOST", "0.0.0.0"),
Port: port,
Env: getenv("APP_ENV", "lab"),
DatabaseURL: dbURL,
CacheURL: cacheURL,
SessionTTL: time.Duration(ttl) * time.Second,
SessionStore: store,
RedisRequired: store == "redis",
InstanceID: instanceID,
S3Endpoint: getenv("S3_ENDPOINT", ""),
S3Region: getenv("S3_REGION", "us-east-1"),
S3Bucket: getenv("S3_BUCKET", "messenger-attachments"),
S3AccessKeyID: s3Access,
S3SecretKey: s3Secret,
MaxUploadBytes: maxUpload,
MaxImagePixels: maxPixels,
S3GCGrace: gcGrace,
}
}
func randomShortID() string {
b := make([]byte, 6)
if _, err := rand.Read(b); err != nil {
return "unknown"
}
return hex.EncodeToString(b)
}
+12
View File
@@ -0,0 +1,12 @@
package domain
const (
AttachmentQueued = "queued"
AttachmentProcessing = "processing"
AttachmentReady = "ready"
AttachmentFailed = "failed"
)
const PipelineVersion = 1
var RequiredVariants = []string{"thumbnail", "crop", "optimized"}
+613
View File
@@ -0,0 +1,613 @@
package domain
import (
"context"
"errors"
"log"
"math/rand"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// CreateAttachmentPending резервирует строку ДО записи в S3.
// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится.
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, stagingKey, filename string) (string, error) {
id := uuid.NewString()
_, err := s.pool.Exec(ctx,
`INSERT INTO attachments (id, chat_id, owner_id, object_key, staging_key, filename,
content_type, size_bytes, sha256, width, height,
pipeline_version, status)
VALUES ($1,$2,$3,$4,$4,$5,'application/octet-stream',0,repeat('0',64),0,0,1,'queued')`,
id, chatID, ownerID, stagingKey, filename)
if err != nil {
return "", classifyDBError(err)
}
return id, nil
}
// MarkAttachmentFailed помечает pending как failed (компенсация).
func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachments SET status='failed', error_code=$2, updated_at=now()
WHERE id=$1 AND status='queued'`, id, code)
return classifyDBError(err)
}
// ScheduleObjectGC ставит object_key в очередь на удаление (grace period).
func (s *Store) ScheduleObjectGC(ctx context.Context, objectKey, reason string) error {
_, err := s.pool.Exec(ctx,
`INSERT INTO attachment_gc (object_key, reason) VALUES ($1,$2)
ON CONFLICT (object_key) DO NOTHING`, objectKey, reason)
return classifyDBError(err)
}
// GetAttachmentForUser возвращает вложение, только если пользователь — участник чата.
func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*Attachment, error) {
var a Attachment
var errCode *string
err := s.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
a.ErrorCode = errCode
a.Variants = []Variant{}
rows, err := s.pool.Query(ctx,
`SELECT variant, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2
AND variant = ANY($3)
ORDER BY variant`,
a.ID, a.PipelineVersion, RequiredVariants)
if err != nil {
return nil, classifyDBError(err)
}
defer rows.Close()
for rows.Next() {
var v Variant
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes,
&v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
return nil, classifyDBError(err)
}
a.Variants = append(a.Variants, v)
}
if err := rows.Err(); err != nil {
return nil, classifyDBError(err)
}
return &a, nil
}
// GetAttachmentObjectKey — только для внутреннего download после проверки ACL.
func (s *Store) GetAttachmentObjectKey(ctx context.Context, id string) (string, error) {
var key string
err := s.pool.QueryRow(ctx, `SELECT object_key FROM attachments WHERE id=$1`, id).Scan(&key)
if errors.Is(err, pgx.ErrNoRows) {
return "", ErrNotFound
}
if err != nil {
return "", classifyDBError(err)
}
return key, nil
}
// ValidateAttachmentIDsForChat проверяет, что все вложения принадлежат чату,
// имеют статус ready и лимит 10. Возвращает ErrInvalid/ErrNotFound.
func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string, ids []string) error {
if len(ids) > 10 {
return ErrInvalid
}
seen := map[string]bool{}
for _, id := range ids {
if seen[id] {
return ErrInvalid
}
seen[id] = true
var status string
err := s.pool.QueryRow(ctx,
`SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`,
id, chatID).Scan(&status)
if err == nil && status == AttachmentFailed {
return ErrInvalid
}
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return classifyDBError(err)
}
}
return nil
}
// ReconcileOrphans: удаляет объекты из attachment_gc старше grace, которых нет в БД,
// и «призрачные» объекты pending старше grace. Вызывается из фонового тикера.
func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Duration) error {
// 1) GC-очередь: удалить объекты в S3, затем строку.
rows, err := s.pool.Query(ctx,
`SELECT object_key FROM attachment_gc WHERE created_at < now() - $1::interval LIMIT 100`,
grace.String())
if err != nil {
return classifyDBError(err)
}
var keys []string
for rows.Next() {
var k string
if err := rows.Scan(&k); err != nil {
rows.Close()
return classifyDBError(err)
}
keys = append(keys, k)
}
rows.Close()
for _, k := range keys {
if err := s3.DeleteObject(ctx, k); err != nil {
continue
}
_, _ = s.pool.Exec(ctx, `DELETE FROM attachment_gc WHERE object_key=$1`, k)
}
// 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет.
prow, err := s.pool.Query(ctx,
`SELECT a.id, a.staging_key FROM attachments a
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id
WHERE a.status='queued' AND j.job_id IS NULL
AND a.created_at < now() - $1::interval LIMIT 100`,
grace.String())
if err != nil {
return classifyDBError(err)
}
type pend struct{ id, key string }
var pendings []pend
for prow.Next() {
var p pend
if err := prow.Scan(&p.id, &p.key); err != nil {
prow.Close()
return classifyDBError(err)
}
pendings = append(pendings, p)
}
prow.Close()
for _, p := range pendings {
_ = s3.DeleteObject(ctx, p.key)
_, _ = s.pool.Exec(ctx, `DELETE FROM attachments WHERE id=$1 AND status='pending'`, p.id)
}
return nil
}
// EnqueueAttachment: атомарно фиксируем метаданные приёма и кладём outbox-строку.
// 202 отвечаем только после успешного COMMIT этой транзакции.
func (s *Store) EnqueueAttachment(
ctx context.Context, attachmentID, contentType string,
size int64, sha string, w, h int, pipelineVersion int,
) (*Attachment, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var a Attachment
err = tx.QueryRow(ctx, `
UPDATE attachments
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
pipeline_version=$7, status='queued', updated_at=now()
WHERE id=$1 AND status='queued'
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at, pipeline_version`,
attachmentID, contentType, size, sha, w, h, pipelineVersion,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes,
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_jobs
(job_id, attachment_id, pipeline_version, status, next_attempt_at)
VALUES ($1,$2,$3,'queued', now())`,
uuid.NewString(), attachmentID, pipelineVersion,
); err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
a.Variants = []Variant{}
return &a, nil
}
// ClaimJob — берём одну доступную задачу под lease.
// Работает и для queued, и для протухшего processing (reaper тем же запросом).
func (s *Store) ClaimJob(ctx context.Context, workerID string, lease time.Duration) (*Job, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var j Job
err = tx.QueryRow(ctx, `
WITH picked AS (
SELECT job_id FROM attachment_jobs
WHERE status IN ('queued','processing')
AND next_attempt_at <= now()
AND (status='queued' OR lease_expires_at < now())
ORDER BY next_attempt_at
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE attachment_jobs aj
SET status='processing',
lease_owner=$1,
lease_expires_at = now() + $2::interval,
updated_at = now()
FROM picked
WHERE aj.job_id = picked.job_id
RETURNING aj.job_id, aj.attachment_id, aj.pipeline_version,
aj.attempt, aj.status, aj.next_attempt_at, aj.last_error_code`,
workerID, lease.String(),
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.Status,
&j.NextAttemptAt, &j.LastError)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
return &j, nil
}
// MarkReady: атомарно пишем все варианты (идемпотентно по ключу) и завершаем job.
// Конкурентная попытка не перезапишет существующий вариант — ON CONFLICT DO NOTHING.
func (s *Store) MarkReady(
ctx context.Context, jobID, attachmentID string, pipelineVersion int,
finalOriginalKey string, variants []Variant,
) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
// Если job уже done — no-op (redelivery после ack).
var status string
err = tx.QueryRow(ctx,
`SELECT status FROM attachment_jobs WHERE job_id=$1 FOR UPDATE`, jobID,
).Scan(&status)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return classifyDBError(err)
}
if status == "done" {
return tx.Commit(ctx)
}
if status == "failed" {
return ErrConflict // нельзя откатывать failed→ready; см. replay
}
// Записать оригинал в final prefix.
if _, err := tx.Exec(ctx,
`UPDATE attachments
SET object_key=$2, status='ready', ready_at=now(), updated_at=now()
WHERE id=$1 AND pipeline_version=$3`,
attachmentID, finalOriginalKey, pipelineVersion,
); err != nil {
return classifyDBError(err)
}
for _, v := range variants {
if v.Name == "original" {
continue // original живёт в attachments.object_key, не в variants
}
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_variants
(attachment_id, pipeline_version, variant, object_key,
content_type, size_bytes, width, height, sha256)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`,
attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name,
v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256,
); err != nil {
return classifyDBError(err)
}
}
if _, err := tx.Exec(ctx,
`UPDATE attachment_jobs
SET status='done', lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID,
); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
// RetryJob: временная ошибка — вернуть в очередь с backoff+jitter.
// exhausted=true — если попытки кончились.
func (s *Store) RetryJob(
ctx context.Context, jobID string, errorCode string,
maxAttempts int, baseBackoff time.Duration,
) (exhausted bool, err error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, classifyDBError(err)
}
defer tx.Rollback(ctx)
var attempt int
var attachmentID string
err = tx.QueryRow(ctx,
`SELECT attempt, attachment_id FROM attachment_jobs
WHERE job_id=$1 FOR UPDATE`, jobID,
).Scan(&attempt, &attachmentID)
if err != nil {
return false, classifyDBError(err)
}
if attempt >= maxAttempts {
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status='failed', last_error_code=$2,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID, errorCode); err != nil {
return false, classifyDBError(err)
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status='failed', failed_at=now(),
error_code=$2, updated_at=now()
WHERE id=$1`, attachmentID, errorCode); err != nil {
return false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
log.Printf("retry job=%s exhausted attempts=%d code=%s",
jobID, attempt, errorCode)
return true, nil
}
// backoff: base * 2^(attempt-1) + jitter
backoff := baseBackoff * time.Duration(1<<(attempt-1))
jitter := time.Duration(rand.Int63n(int64(backoff / 4)))
next := time.Now().Add(backoff + jitter)
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET attempt = attempt + 1,
status = 'queued',
last_error_code=$2,
next_attempt_at=$3,
published_at=NULL,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID, errorCode, next); err != nil {
return false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
// Логируем зафиксированный retry — используется в student-tests/backoff.py
log.Printf("retry job=%s attempt=%d next_attempt=%s backoff=%s jitter=%s code=%s",
jobID, attempt+1, next.UTC().Format(time.RFC3339Nano),
backoff, jitter, errorCode)
return false, nil
}
// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox.
func (s *Store) RequeueStuck(ctx context.Context) (int64, error) {
tag, err := s.pool.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
next_attempt_at = now(), published_at = NULL, updated_at = now()
WHERE status='processing' AND lease_expires_at < now()`)
if err != nil {
return 0, classifyDBError(err)
}
return tag.RowsAffected(), nil
}
// ReplayFailed — операторский replay (уровень 4, CLI/endpoint).
func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', attempt=1, next_attempt_at=now(),
published_at=NULL,
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
if err != nil {
return classifyDBError(err)
}
if tag.RowsAffected() == 0 {
return ErrConflict
}
if _, err := tx.Exec(ctx,
`UPDATE attachments SET status='queued', error_code=NULL, failed_at=NULL,
updated_at=now()
WHERE id=$1 AND status='failed'`, attachmentID); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
// ClaimJobByID — берёт конкретную job под lease. Нужен для redelivery:
// если worker упал после publish, повторно доставленное сообщение должно
// либо взять ту же работу (lease протух), либо быть no-op (status=done).
func (s *Store) ClaimJobByID(ctx context.Context, jobID, workerID string, lease time.Duration) (*Job, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var j Job
err = tx.QueryRow(ctx, `
UPDATE attachment_jobs
SET status='processing',
lease_owner=$2,
lease_expires_at = now() + $3::interval,
updated_at = now()
WHERE job_id=$1
AND status IN ('queued','processing')
AND (status='queued' OR lease_expires_at < now())
RETURNING job_id, attachment_id, pipeline_version, attempt,
status, next_attempt_at, last_error_code`,
jobID, workerID, lease.String(),
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt,
&j.Status, &j.NextAttemptAt, &j.LastError)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
return &j, nil
}
// GetStagingKey — доверенные метаданные для worker’а (не из сообщения брокера).
func (s *Store) GetStagingKey(ctx context.Context, attachmentID string) (string, string, error) {
var key, ct string
err := s.pool.QueryRow(ctx,
`SELECT staging_key, content_type FROM attachments WHERE id=$1`, attachmentID,
).Scan(&key, &ct)
if errors.Is(err, pgx.ErrNoRows) {
return "", "", ErrNotFound
}
if err != nil {
return "", "", classifyDBError(err)
}
return key, ct, nil
}
// VariantsExist — no-op-проверка на повторную доставку уже готовой работы.
func (s *Store) VariantsExist(ctx context.Context, attachmentID string, pv int) (bool, error) {
var n int
err := s.pool.QueryRow(ctx,
`SELECT count(*) FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2`,
attachmentID, pv,
).Scan(&n)
if err != nil {
return false, classifyDBError(err)
}
return n >= len(RequiredVariants), nil
}
// MarkFailedPermanent — терминальный failed без retry (постоянная ошибка).
func (s *Store) MarkFailedPermanent(ctx context.Context, jobID, attachmentID, code string) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx,
`UPDATE attachment_jobs SET status='failed', last_error_code=$2,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1 AND status='processing'`, jobID, code); err != nil {
return classifyDBError(err)
}
if _, err := tx.Exec(ctx,
`UPDATE attachments SET status='failed', error_code=$2,
failed_at=now(), updated_at=now()
WHERE id=$1`, attachmentID, code); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
type UnpublishedJob struct {
JobID string
AttachmentID string
PipelineVersion int
Attempt int
}
// SelectUnpublishedJobs — выборка outbox’ом. Транзакция не нужна:
// published_at ставим отдельным UPDATE после XADD; повторная публикация
// допустима (worker идемпотентен).
func (s *Store) SelectUnpublishedJobs(ctx context.Context, limit int) ([]UnpublishedJob, error) {
rows, err := s.pool.Query(ctx, `
SELECT job_id, attachment_id, pipeline_version, attempt
FROM attachment_jobs
WHERE published_at IS NULL AND status='queued'
AND next_attempt_at <= now()
ORDER BY next_attempt_at
LIMIT $1`, limit)
if err != nil {
return nil, classifyDBError(err)
}
defer rows.Close()
var out []UnpublishedJob
for rows.Next() {
var u UnpublishedJob
if err := rows.Scan(&u.JobID, &u.AttachmentID, &u.PipelineVersion, &u.Attempt); err != nil {
return nil, classifyDBError(err)
}
out = append(out, u)
}
return out, rows.Err()
}
// MarkPublished — фиксируем факт XADD (publisher confirm).
func (s *Store) MarkPublished(ctx context.Context, jobID string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachment_jobs SET published_at=now(),
publish_attempts=publish_attempts+1, updated_at=now()
WHERE job_id=$1`, jobID)
return classifyDBError(err)
}
// BumpPublishAttempt — XADD упал; фиксируем причину, повторим в следующем цикле.
func (s *Store) BumpPublishAttempt(ctx context.Context, jobID, code string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachment_jobs SET publish_attempts=publish_attempts+1,
last_error_code=$2, updated_at=now()
WHERE job_id=$1`, jobID, code)
return classifyDBError(err)
}
func (s *Store) GetVariantKey(ctx context.Context, attachmentID string, pv int, variant string) (string, string, error) {
var key, ct string
err := s.pool.QueryRow(ctx,
`SELECT object_key, content_type FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2 AND variant=$3`,
attachmentID, pv, variant).Scan(&key, &ct)
if errors.Is(err, pgx.ErrNoRows) {
return "", "", ErrNotFound
}
if err != nil {
return "", "", classifyDBError(err)
}
return key, ct, nil
}
+80
View File
@@ -0,0 +1,80 @@
package domain
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
const (
StreamName = "attachment.jobs"
ConsumerGroup = "workers"
)
type JobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
}
type Broker struct{ rdb *redis.Client }
func NewBroker(ctx context.Context, url string) (*Broker, error) {
opt, err := redis.ParseURL(url)
if err != nil {
return nil, err
}
rdb := redis.NewClient(opt)
if err := rdb.Ping(ctx).Err(); err != nil {
return nil, err
}
// Группа создаётся идемпотентно.
err = rdb.XGroupCreateMkStream(ctx, StreamName, ConsumerGroup, "0").Err()
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
return nil, err
}
return &Broker{rdb: rdb}, nil
}
func (b *Broker) Publish(ctx context.Context, m JobMessage) (string, error) {
buf, err := json.Marshal(m)
if err != nil {
return "", err
}
return b.rdb.XAdd(ctx, &redis.XAddArgs{
Stream: StreamName,
Values: map[string]any{"payload": buf},
}).Result()
}
// Consume — блокирующее чтение из consumer group; ack делает вызывающий.
func (b *Broker) Consume(ctx context.Context, consumer string) ([]redis.XMessage, error) {
res, err := b.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: ConsumerGroup, Consumer: consumer,
Streams: []string{StreamName, ">"},
Count: 1, Block: 5 * time.Second,
}).Result()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err != nil {
return nil, err
}
if len(res) == 0 {
return nil, nil
}
return res[0].Messages, nil
}
func (b *Broker) Ack(ctx context.Context, id string) error {
return b.rdb.XAck(ctx, StreamName, ConsumerGroup, id).Err()
}
+91
View File
@@ -0,0 +1,91 @@
package domain
import (
"context"
"encoding/json"
"time"
"github.com/redis/go-redis/v9"
)
// SessionCache — необязательный кэш сессий поверх Redis.
// Источник истины — PostgreSQL. Кэш ускоряет проверку токена.
type SessionCache struct {
rdb *redis.Client
}
func NewSessionCache(ctx context.Context, url string) (*SessionCache, error) {
opt, err := redis.ParseURL(url)
if err != nil {
return nil, err
}
// Жёсткие короткие таймауты: Redis — кэш, а не критический путь.
opt.DialTimeout = 1 * time.Second
opt.ReadTimeout = 500 * time.Millisecond
opt.WriteTimeout = 500 * time.Millisecond
opt.MaxRetries = 0 // без автоматических ретраев: одна попытка — и уходим в БД
opt.PoolTimeout = 1 * time.Second
rdb := redis.NewClient(opt)
// Проверка связи при старте — короткая.
c, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
if err := rdb.Ping(c).Err(); err != nil {
_ = rdb.Close()
return nil, err
}
return &SessionCache{rdb: rdb}, nil
}
func (c *SessionCache) userKey(userID string) string { return "user:" + userID }
// opCtx возвращает контекст с коротким таймаутом для каждой операции.
// Не наследуем HTTP-контекст напрямую, чтобы не блокировать ответ клиенту.
func (c *SessionCache) opCtx() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 500*time.Millisecond)
}
// SetUser кладёт публичные данные пользователя в кэш с TTL до конца сессии.
func (c *SessionCache) SetUser(ctx context.Context, userID string, u User, expires time.Time) {
ttl := time.Until(expires)
if ttl <= 0 {
return
}
b, err := json.Marshal(u)
if err != nil {
return
}
opCtx, cancel := c.opCtx()
defer cancel()
_ = c.rdb.Set(opCtx, c.userKey(userID), b, ttl).Err()
}
func (c *SessionCache) GetUser(ctx context.Context, userID string) (User, bool) {
opCtx, cancel := c.opCtx()
defer cancel()
b, err := c.rdb.Get(opCtx, c.userKey(userID)).Bytes()
if err != nil {
return User{}, false
}
var u User
if err := json.Unmarshal(b, &u); err != nil {
return User{}, false
}
return u, true
}
// DelUser — на случай удаления пользователя (в lab2 не вызывается).
func (c *SessionCache) DelUser(ctx context.Context, userID string) error {
opCtx, cancel := c.opCtx()
defer cancel()
return c.rdb.Del(opCtx, c.userKey(userID)).Err()
}
func (c *SessionCache) Ping(ctx context.Context) error {
c2, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
return c.rdb.Ping(c2).Err()
}
func (c *SessionCache) Close() error { return c.rdb.Close() }
+75
View File
@@ -0,0 +1,75 @@
package domain
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// IdempotencyRecord — сохранённый результат POST.
type IdempotencyRecord struct {
StatusCode int
Response json.RawMessage
}
// LookupIdempotency возвращает запись, если ключ уже использован.
// Второй аргумент — совпадает ли хеш тела с сохранённым.
func (s *Store) LookupIdempotency(ctx context.Context, userID, method, path, key, bodyHash string) (*IdempotencyRecord, bool, error) {
var (
storedHash string
rec IdempotencyRecord
)
err := s.pool.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method=$2 AND path=$3 AND key=$4`,
userID, method, path, key,
).Scan(&storedHash, &rec.StatusCode, &rec.Response)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil {
return nil, false, classifyDBError(err)
}
if storedHash != bodyHash {
return nil, true, ErrConflict
}
return &rec, true, nil
}
// SaveIdempotency пишет результат. Конфликт PK означает, что параллельный
// запрос успел раньше — вызывающий должен перечитать через LookupIdempotency.
func (s *Store) SaveIdempotency(ctx context.Context, userID, method, path, key, bodyHash string, status int, resp json.RawMessage) error {
_, err := s.pool.Exec(ctx,
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
userID, method, path, key, bodyHash, status, resp)
return classifyDBError(err)
}
// HashBody — стабильный хеш JSON независимо от порядка ключей.
func HashBody(raw []byte) (string, error) {
var v any
if err := json.Unmarshal(raw, &v); err != nil {
return "", err
}
normalized, err := json.Marshal(v) // map keys сортируются
if err != nil {
return "", err
}
sum := sha256.Sum256(normalized)
return hex.EncodeToString(sum[:]), nil
}
// CleanupIdempotency — можно звать из фонового тикера (опционально).
func (s *Store) CleanupIdempotency(ctx context.Context, olderThan time.Duration) error {
_, err := s.pool.Exec(ctx,
`DELETE FROM idempotency_keys WHERE created_at < now() - $1::interval`,
olderThan.String())
return err
}
+13
View File
@@ -0,0 +1,13 @@
package domain
import "time"
type Job struct {
JobID string
AttachmentID string
PipelineVersion int
Attempt int
Status string
NextAttemptAt time.Time
LastError *string
}
+94
View File
@@ -0,0 +1,94 @@
package domain
import (
"errors"
"time"
)
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt time.Time `json:"created_at"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt time.Time `json:"created_at"`
}
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type MessagePage struct {
Items []Message `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type ChatPage struct {
Items []Chat `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type Session struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt time.Time `json:"expires_at"`
User User `json:"user"`
}
type ErrorBody struct {
Error ErrorDetail `json:"error"`
}
type ErrorDetail struct {
Code string `json:"code"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
ObjectKey string `json:"-"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
PipelineVersion int `json:"-"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
var (
ErrNotFound = errors.New("not found")
ErrConflict = errors.New("conflict")
ErrInvalid = errors.New("invalid request")
ErrUnauthorized = errors.New("unauthorized")
ErrTimeout = errors.New("timeout")
ErrUnavailable = errors.New("unavailable")
ErrTooLarge = errors.New("payload too large")
ErrUnsupported = errors.New("unsupported media type")
ErrNotImage = errors.New("not a valid image")
)
+619
View File
@@ -0,0 +1,619 @@
package domain
import (
"context"
"errors"
"fmt"
"strings"
"time"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"encoding/json"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/crypto/argon2"
)
type Store struct {
pool *pgxpool.Pool
}
func NewStore(ctx context.Context, dsn string) (*Store, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, classifyDBError(err)
}
cfg.MaxConns = 10
cfg.MinConns = 1
cfg.MaxConnLifetime = time.Hour
cfg.MaxConnIdleTime = 5 * time.Minute
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
cfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
_, err := conn.Exec(ctx, `
SET statement_timeout = '5s';
SET lock_timeout = '2s';
SET idle_in_transaction_session_timeout = '5s';
`)
return err
}
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, classifyDBError(err)
}
ctxPing, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := pool.Ping(ctxPing); err != nil {
pool.Close()
return nil, classifyDBError(err)
}
return &Store{pool: pool}, nil
}
func (s *Store) Close() { s.pool.Close() }
func (s *Store) Ping(ctx context.Context) error {
c, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
return s.pool.Ping(c)
}
// Argon2id параметры — OWASP baseline: m=19456 KiB (19 MiB), t=2, p=1.
// Хеш кодируется в формате PHC: $argon2id$v=19$m=19456,t=2,p=1$<salt>$<hash>
const (
argonMemory uint32 = 19456
argonTime uint32 = 2
argonThreads uint8 = 1
argonKeyLen uint32 = 32
argonSaltLen = 16
)
func hashPassword(password string) (string, error) {
salt := make([]byte, argonSaltLen)
if _, err := rand.Read(salt); err != nil {
return "", err
}
key := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
b64Salt := base64.RawStdEncoding.EncodeToString(salt)
b64Key := base64.RawStdEncoding.EncodeToString(key)
// PHC string; параметры можно будет менять в будущем, старые хеши останутся валидными.
return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$%s$%s",
argonMemory, argonTime, argonThreads, b64Salt, b64Key), nil
}
func verifyPassword(password, encoded string) bool {
// Минимальный парсер PHC-строки.
parts := strings.Split(encoded, "$")
if len(parts) != 6 || parts[1] != "argon2id" {
return false
}
var mem, time uint32
var threads uint8
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &time, &threads); err != nil {
return false
}
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
if err != nil {
return false
}
wantKey, err := base64.RawStdEncoding.DecodeString(parts[5])
if err != nil {
return false
}
gotKey := argon2.IDKey([]byte(password), salt, time, mem, threads, uint32(len(wantKey)))
return subtle.ConstantTimeCompare(gotKey, wantKey) == 1
}
// --- Users ---
func (s *Store) CreateUser(ctx context.Context, username, displayName, password string) (*User, error) {
hash, err := hashPassword(password)
if err != nil {
return nil, classifyDBError(err)
}
id := uuid.NewString()
var u User
err = s.pool.QueryRow(ctx,
`INSERT INTO users (id, username, display_name, password_hash)
VALUES ($1,$2,$3,$4)
RETURNING id, username, display_name, created_at`,
id, username, displayName, string(hash),
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt)
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
return nil, ErrConflict
}
return nil, classifyDBError(err)
}
return &u, nil
}
func (s *Store) GetUserByID(ctx context.Context, id string) (*User, error) {
var u User
err := s.pool.QueryRow(ctx,
`SELECT id, username, display_name, created_at FROM users WHERE id=$1`, id,
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
return &u, nil
}
// VerifyPassword возвращает пользователя, если пароль верный; иначе ErrUnauthorized.
func (s *Store) VerifyPassword(ctx context.Context, username, password string) (*User, error) {
var u User
var hash string
err := s.pool.QueryRow(ctx,
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username=$1`,
username,
).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt, &hash)
if errors.Is(err, pgx.ErrNoRows) {
// Тратим сопоставимое время на проверку фиктивного хеша,
// чтобы не различать "нет пользователя" и "неверный пароль" по таймингу.
_ = verifyPassword(password, "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
return nil, ErrUnauthorized
}
if err != nil {
return nil, classifyDBError(err)
}
if !verifyPassword(password, hash) {
return nil, ErrUnauthorized
}
return &u, nil
}
// --- Chats ---
func (s *Store) CreateChat(ctx context.Context, title string, memberIDs []string, creatorID string) (*Chat, error) {
// Проверка входных данных.
seen := map[string]bool{}
for _, id := range memberIDs {
if seen[id] {
return nil, ErrInvalid
}
seen[id] = true
}
if !seen[creatorID] {
return nil, ErrInvalid
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
// Проверяем, что все участники существуют.
for id := range seen {
var exists bool
if err := tx.QueryRow(ctx, `SELECT true FROM users WHERE id=$1`, id).Scan(&exists); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrInvalid
}
return nil, classifyDBError(err)
}
}
chatID := uuid.NewString()
var c Chat
err = tx.QueryRow(ctx,
`INSERT INTO chats (id, title) VALUES ($1,$2) RETURNING id, title, created_at`,
chatID, title,
).Scan(&c.ID, &c.Title, &c.CreatedAt)
if err != nil {
return nil, classifyDBError(err)
}
for id := range seen {
if _, err := tx.Exec(ctx,
`INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2)`,
c.ID, id,
); err != nil {
return nil, classifyDBError(err)
}
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
// Порядок member_ids — как в запросе.
c.MemberIDs = append([]string(nil), memberIDs...)
return &c, nil
}
func (s *Store) GetChat(ctx context.Context, chatID, userID string) (*Chat, error) {
var exists bool
err := s.pool.QueryRow(ctx,
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
chatID, userID,
).Scan(&exists)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
var c Chat
err = s.pool.QueryRow(ctx,
`SELECT id, title, created_at FROM chats WHERE id=$1`, chatID,
).Scan(&c.ID, &c.Title, &c.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
rows, err := s.pool.Query(ctx,
`SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chatID)
if err != nil {
return nil, classifyDBError(err)
}
defer rows.Close()
for rows.Next() {
var uid string
if err := rows.Scan(&uid); err != nil {
return nil, classifyDBError(err)
}
c.MemberIDs = append(c.MemberIDs, uid)
}
return &c, rows.Err()
}
// ListChats возвращает чаты пользователя с keyset-пагинацией по (created_at, id).
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cursor string) ([]Chat, *string, error) {
var rows pgx.Rows
var err error
if cursor == "" {
rows, err = s.pool.Query(ctx,
`SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = $1
ORDER BY c.created_at, c.id
LIMIT $2`, userID, limit+1)
} else {
var cCreated time.Time
if err := s.pool.QueryRow(ctx,
`SELECT created_at FROM chats WHERE id=$1`, cursor,
).Scan(&cCreated); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, ErrInvalid
}
return nil, nil, classifyDBError(err)
}
rows, err = s.pool.Query(ctx,
`SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = $1
AND (c.created_at, c.id) > ($2, $3)
ORDER BY c.created_at, c.id
LIMIT $4`, userID, cCreated, cursor, limit+1)
}
if err != nil {
return nil, nil, classifyDBError(err)
}
defer rows.Close()
var chats []Chat
for rows.Next() {
var c Chat
if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
return nil, nil, classifyDBError(err)
}
chats = append(chats, c)
}
if err := rows.Err(); err != nil {
return nil, nil, classifyDBError(err)
}
var next *string
if len(chats) > limit {
chats = chats[:limit]
last := chats[len(chats)-1].ID
next = &last
}
if chats == nil {
chats = []Chat{}
}
// Догружаем участников.
for i := range chats {
mrows, err := s.pool.Query(ctx,
`SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id`, chats[i].ID)
if err != nil {
return nil, nil, classifyDBError(err)
}
for mrows.Next() {
var uid string
if err := mrows.Scan(&uid); err != nil {
mrows.Close()
return nil, nil, classifyDBError(err)
}
chats[i].MemberIDs = append(chats[i].MemberIDs, uid)
}
mrows.Close()
}
return chats, next, nil
}
// --- Messages ---
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (*Message, error) {
// Проверка членства + вставка в одной транзакции.
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var ok bool
if err := tx.QueryRow(ctx,
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
chatID, senderID,
).Scan(&ok); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return nil, classifyDBError(err)
}
var m Message
var attach []string
err = tx.QueryRow(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
VALUES ($1,$2,$3,$4,$5)
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
uuid.NewString(), chatID, senderID, text, []string{},
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
if err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
m.AttachmentIDs = attach
if m.AttachmentIDs == nil {
m.AttachmentIDs = []string{}
}
return &m, nil
}
// ListMessages — keyset-пагинация по (created_at, id).
// Курсор — это id последнего элемента предыдущей страницы; сам id уникален,
// поэтому используем его для поиска (created_at, id) в БД.
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cursor string) ([]Message, *string, error) {
var rows pgx.Rows
var err error
if cursor == "" {
rows, err = s.pool.Query(ctx,
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
FROM messages
WHERE chat_id=$1
ORDER BY created_at, id
LIMIT $2`, chatID, limit+1)
} else {
// Определяем позицию курсора.
var cCreated time.Time
if err := s.pool.QueryRow(ctx,
`SELECT created_at FROM messages WHERE id=$1 AND chat_id=$2`,
cursor, chatID,
).Scan(&cCreated); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, ErrInvalid
}
return nil, nil, classifyDBError(err)
}
rows, err = s.pool.Query(ctx,
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
FROM messages
WHERE chat_id=$1
AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT $4`, chatID, cCreated, cursor, limit+1)
}
if err != nil {
return nil, nil, classifyDBError(err)
}
defer rows.Close()
var items []Message
for rows.Next() {
var m Message
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
return nil, nil, classifyDBError(err)
}
if m.AttachmentIDs == nil {
m.AttachmentIDs = []string{}
}
items = append(items, m)
}
if err := rows.Err(); err != nil {
return nil, nil, classifyDBError(err)
}
var next *string
if len(items) > limit {
items = items[:limit]
last := items[len(items)-1].ID
next = &last
}
if items == nil {
items = []Message{}
}
return items, next, nil
}
func classifyDBError(err error) error {
if err == nil {
return nil
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
switch pgErr.Code {
case "57014": // query_canceled (statement_timeout)
return ErrTimeout
case "55P03": // lock_not_available (lock_timeout)
return ErrTimeout
case "40001": // serialization_failure
return ErrUnavailable
case "53300": // too_many_connections
return ErrUnavailable
case "22P02", "22007", "22023": // invalid_text_representation / invalid_datetime_format / invalid_parameter_value
return ErrInvalid
}
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
return ErrTimeout
}
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
return err
}
// AddMessageIdempotent — атомарно проверяет членство, вставляет сообщение
// и сохраняет результат идемпотентности в одной транзакции.
// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true.
func (s *Store) AddMessageIdempotent(
ctx context.Context,
chatID, senderID, text string,
attachmentIDs []string,
key, bodyHash string,
) (msg *Message, rec *IdempotencyRecord, replay bool, err error) {
path := "/api/v1/chats/" + chatID + "/messages"
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
if err != nil {
return nil, nil, false, classifyDBError(err)
}
defer tx.Rollback(ctx)
// 1) Членство.
var ok bool
if err := tx.QueryRow(ctx,
`SELECT true FROM chat_members WHERE chat_id=$1 AND user_id=$2`,
chatID, senderID,
).Scan(&ok); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, false, ErrNotFound
}
return nil, nil, false, classifyDBError(err)
}
// Вложения должны принадлежать этому чату и быть ready.
if len(attachmentIDs) > 10 {
return nil, nil, false, ErrInvalid
}
seen := map[string]bool{}
for _, aid := range attachmentIDs {
if seen[aid] {
return nil, nil, false, ErrInvalid
}
seen[aid] = true
var (
status string
objectKey string
)
// Вложение из этого же чата; не обязательно ready — клиент увидит картинку,
// когда worker завершит обработку. failed — принимаем: сообщение уже создано,
// UI покажет «failed» через GET /attachments/{id}.
if err := tx.QueryRow(ctx,
`SELECT status, object_key FROM attachments WHERE id=$1 AND chat_id=$2`,
aid, chatID).Scan(&status, &objectKey); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, false, ErrNotFound
}
return nil, nil, false, classifyDBError(err)
}
if status == AttachmentFailed {
// Нельзя прикреплять заведомо неудачные вложения — иначе UI будет
// навсегда показывать сломанную картинку без пути к replay.
return nil, nil, false, ErrInvalid
}
}
// 2) Уже был такой ключ?
var storedHash string
var storedStatus int
var storedResp json.RawMessage
err = tx.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
senderID, path, key,
).Scan(&storedHash, &storedStatus, &storedResp)
switch {
case err == nil:
if storedHash != bodyHash {
return nil, nil, false, ErrConflict
}
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
case !errors.Is(err, pgx.ErrNoRows):
return nil, nil, false, classifyDBError(err)
}
// 3) Вставка сообщения.
var m Message
var attach []string
err = tx.QueryRow(ctx,
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
VALUES ($1,$2,$3,$4,$5)
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
uuid.NewString(), chatID, senderID, text, attachmentIDs,
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
if err != nil {
// Гонка на одном ключе в разных репликах: unique violation в idempotency_keys
// маловероятен здесь, но serialization failure возможен.
return nil, nil, false, classifyDBError(err)
}
m.AttachmentIDs = attach
if m.AttachmentIDs == nil {
m.AttachmentIDs = []string{}
}
// 4) Сохранение ответа.
respJSON, _ := json.Marshal(m)
_, err = tx.Exec(ctx,
`INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
VALUES ($1,'POST',$2,$3,$4,201,$5)`,
senderID, path, key, bodyHash, respJSON)
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
// Кто-то параллельно успел вставить: перечитываем в этой же tx.
if err2 := tx.QueryRow(ctx,
`SELECT request_hash, status_code, response
FROM idempotency_keys
WHERE user_id=$1 AND method='POST' AND path=$2 AND key=$3`,
senderID, path, key,
).Scan(&storedHash, &storedStatus, &storedResp); err2 == nil {
if storedHash != bodyHash {
return nil, nil, false, ErrConflict
}
return nil, &IdempotencyRecord{StatusCode: storedStatus, Response: storedResp}, true, nil
}
}
return nil, nil, false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, nil, false, classifyDBError(err)
}
return &m, nil, false, nil
}
+258
View File
@@ -0,0 +1,258 @@
package domain
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type S3Store struct {
client *minio.Client
bucket string
maxBytes int64
maxPixels int64
tr *http.Transport
}
func NewS3Store(ctx context.Context, endpoint, region, bucket, access, secret string, maxBytes, maxPixels int64) (*S3Store, error) {
if endpoint == "" || access == "" || secret == "" || bucket == "" {
return nil, errors.New("s3 configuration incomplete")
}
tr := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 30 * time.Second,
DialContext: (&net.Dialer{
Timeout: 3 * time.Second, // Быстрый отказ при недоступности
KeepAlive: 30 * time.Second,
}).DialContext,
TLSHandshakeTimeout: 3 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: strings.HasPrefix(endpoint, "https://"),
Region: region,
Transport: tr,
})
if err != nil {
return nil, err
}
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if _, err := cli.BucketExists(pingCtx, bucket); err != nil {
return nil, fmt.Errorf("s3 bucket check: %w", err)
}
return &S3Store{client: cli, bucket: bucket, maxBytes: maxBytes, maxPixels: maxPixels, tr: tr}, nil
}
type UploadResult struct {
ObjectKey string
ContentType string
SizeBytes int64
SHA256 string
Width int
Height int
}
// PutValidatedImage читает тело потоково во временный файл на диске,
// проверяет размер, MIME, пиксели и затем стримит в S3.
// Это гарантирует, что память не растёт с размером файла.
func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body io.Reader) (*UploadResult, error) {
// 1. Создаём временный файл.
tmpFile, err := os.CreateTemp("", "upload-*")
if err != nil {
return nil, err
}
tmpPath := tmpFile.Name()
defer func() {
tmpFile.Close()
os.Remove(tmpPath)
}()
// 2. Копируем тело в файл с ограничением.
limited := io.LimitReader(body, s.maxBytes+1)
size, err := io.Copy(tmpFile, limited)
if err != nil {
return nil, err
}
if size > s.maxBytes {
return nil, ErrTooLarge
}
if size == 0 {
return nil, ErrUnsupported
}
// 3. Проверяем MIME по первым 512 байтам.
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return nil, err
}
header := make([]byte, 512)
n, err := io.ReadFull(tmpFile, header)
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
return nil, err
}
header = header[:n]
ct := http.DetectContentType(header)
if ct != "image/png" && ct != "image/jpeg" {
return nil, ErrUnsupported
}
// 4. Проверяем заголовок изображения (размеры).
// DecodeConfig читает только начало, но мы используем файл, поэтому это безопасно.
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return nil, err
}
cfg, _, err := image.DecodeConfig(tmpFile)
if err != nil {
return nil, ErrNotImage
}
if int64(cfg.Width)*int64(cfg.Height) > s.maxPixels {
return nil, ErrNotImage
}
// 5. Считаем SHA-256 и стримим файл в S3.
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return nil, err
}
var hasher hash.Hash = sha256.New()
tee := io.TeeReader(tmpFile, hasher)
putCtx, cancelPut := context.WithTimeout(ctx, 5*time.Second)
defer cancelPut()
_, err = s.client.PutObject(putCtx, s.bucket, objectKey, tee, size,
minio.PutObjectOptions{ContentType: ct})
if err != nil {
return nil, err
}
sha := hex.EncodeToString(hasher.Sum(nil))
return &UploadResult{
ObjectKey: objectKey,
ContentType: ct,
SizeBytes: size,
SHA256: sha,
Width: cfg.Width,
Height: cfg.Height,
}, nil
}
const (
StagingPrefix = "staging/"
FinalPrefix = "final/"
)
// MoveToFinal: копирование staging→final и удаление staging.
// Используем CopyObject, чтобы не тянуть байты в память. Если MinIO не умеет
// server-side copy — используйте compose? MinIO умеет S3 CopyObject.
func (s *S3Store) MoveToFinal(ctx context.Context, stagingKey, finalKey string) error {
src := minio.CopySrcOptions{Bucket: s.bucket, Object: stagingKey}
dst := minio.CopyDestOptions{Bucket: s.bucket, Object: finalKey}
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
return err
}
return s.DeleteObject(ctx, stagingKey)
}
func (s *S3Store) PutBytes(ctx context.Context, key, contentType string, data []byte) (string, error) {
h := sha256.Sum256(data)
_, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType})
if err != nil {
return "", err
}
return hex.EncodeToString(h[:]), nil
}
// FetchStagingForProcessing — читает staging в память с жёстким лимитом.
// maxBytes гарантирует, что worker не съест RAM на «злом» файле.
func (s *S3Store) FetchStagingForProcessing(ctx context.Context, key string) ([]byte, string, error) {
rc, ct, size, err := s.GetObject(ctx, key)
if err != nil {
return nil, "", err
}
defer rc.Close()
if size > s.maxBytes {
return nil, "", ErrTooLarge
}
buf, err := io.ReadAll(rc)
if err != nil {
return nil, "", err
}
return buf, ct, nil
}
func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) {
// Общий дедлайн на все попытки — 6 секунд.
overallCtx, overallCancel := context.WithTimeout(ctx, 6*time.Second)
defer overallCancel()
var lastErr error
for attempt := 0; attempt < 2; attempt++ {
if attempt > 0 {
time.Sleep(500 * time.Millisecond)
s.tr.CloseIdleConnections()
}
attemptCtx, cancel := context.WithTimeout(overallCtx, 3*time.Second)
obj, err := s.client.GetObject(attemptCtx, s.bucket, objectKey, minio.GetObjectOptions{})
if err != nil {
cancel()
lastErr = err
continue
}
info, err := obj.Stat()
if err != nil {
obj.Close()
cancel()
if minio.ToErrorResponse(err).Code == "NoSuchKey" {
return nil, "", 0, ErrNotFound
}
lastErr = err
continue
}
data, err := io.ReadAll(obj)
obj.Close()
cancel()
if err != nil {
lastErr = err
continue
}
if int64(len(data)) != info.Size {
lastErr = fmt.Errorf("size mismatch: got %d, expected %d", len(data), info.Size)
continue
}
return io.NopCloser(bytes.NewReader(data)), info.ContentType, info.Size, nil
}
s.tr.CloseIdleConnections()
return nil, "", 0, fmt.Errorf("s3 get failed after retries: %w", lastErr)
}
func (s *S3Store) DeleteObject(ctx context.Context, objectKey string) error {
return s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{})
}
+108
View File
@@ -0,0 +1,108 @@
package domain
import (
"context"
"crypto/rand"
"encoding/base64"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
type SessionStore struct {
store *Store
cache *SessionCache
ttl time.Duration
}
func NewSessionStore(store *Store, cache *SessionCache, ttl time.Duration) *SessionStore {
return &SessionStore{store: store, cache: cache, ttl: ttl}
}
func newToken() (string, error) {
b := make([]byte, 32) // 256 бит
if _, err := rand.Read(b); err != nil {
return "", classifyDBError(err)
}
return base64.RawURLEncoding.EncodeToString(b), nil
}
// Create — пишем сессию в БД (источник истины) и кэшируем публичные данные User.
func (s *SessionStore) Create(ctx context.Context, user User) (*Session, error) {
token, err := newToken()
if err != nil {
return nil, classifyDBError(err)
}
expires := time.Now().UTC().Add(s.ttl)
if _, err := s.store.pool.Exec(ctx,
`INSERT INTO sessions (token, user_id, expires_at) VALUES ($1,$2,$3)`,
token, user.ID, expires); err != nil {
return nil, classifyDBError(err)
}
if s.cache != nil {
s.cache.SetUser(ctx, user.ID, user, expires)
}
return &Session{
Token: token,
TokenType: "Bearer",
ExpiresAt: expires,
User: user,
}, nil
}
// Lookup ВСЕГДА обращается к PostgreSQL за фактом существования сессии.
// Кэш используется только для публичных данных User.
func (s *SessionStore) Lookup(ctx context.Context, token string) (*User, error) {
var (
userID string
expiresAt time.Time
)
err := s.store.pool.QueryRow(ctx,
`SELECT user_id, expires_at FROM sessions WHERE token=$1`, token,
).Scan(&userID, &expiresAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrUnauthorized
}
if err != nil {
return nil, classifyDBError(err)
}
if time.Now().UTC().After(expiresAt) {
_, _ = s.store.pool.Exec(ctx, `DELETE FROM sessions WHERE token=$1`, token)
return nil, ErrUnauthorized
}
if s.cache != nil {
if u, ok := s.cache.GetUser(ctx, userID); ok {
return &u, nil
}
}
u, err := s.store.GetUserByID(ctx, userID)
if err != nil {
return nil, ErrUnauthorized
}
if s.cache != nil {
s.cache.SetUser(ctx, userID, *u, expiresAt)
}
return u, nil
}
func (s *SessionStore) Revoke(ctx context.Context, token string) error {
var userID string
err := s.store.pool.QueryRow(ctx,
`DELETE FROM sessions WHERE token=$1 RETURNING user_id`, token,
).Scan(&userID)
if errors.Is(err, pgx.ErrNoRows) {
// Сессии уже нет — с точки зрения клиента токен отозван.
return nil
}
if err != nil {
return classifyDBError(err)
}
if s.cache != nil {
// Кэш User не критичен, но полезно сбросить.
_ = s.cache.DelUser(ctx, userID)
}
return nil
}
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""В Stream нет filename/URL/секретов — только id/version/attempt."""
import json, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
upload(c, a, ch, content)
raw = docker('exec', '-T', 'redis', 'redis-cli', '--raw', 'XRANGE', 'attachment.jobs', '-', '+')
ok('stream sample:\n' + raw)
lower = raw.lower()
for bad in ('filename', 'sample.png', 'staging/', 'final/', 'http', 'secret'):
check(bad not in lower, 'broker payload contains forbidden token: %s' % bad)
ok('broker payload has only ids/version/attempt')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+73
View File
@@ -0,0 +1,73 @@
"""Shared helpers for student lab5 tests; no tokens are logged."""
import base64, json, os, subprocess, sys, time, uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, Failure, check # noqa: E402
def fail(msg):
print('FAIL:', msg)
raise SystemExit(1)
def ok(msg='PASS'):
print(msg)
def docker(*args, check_rc=True):
"""Run docker compose <args>; return stdout. Raises if rc!=0 and check_rc."""
rc = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True)
if check_rc and rc.returncode != 0:
raise Failure('docker compose %s: %s' % (' '.join(args), rc.stderr.strip()))
return rc.stdout
def psql(sql):
return docker('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-tAc', sql).strip()
def upload(c, actor, chat, content, filename='sample.png', ct='image/png'):
return c.call('POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
202, actor=actor, raw=content, headers={'Content-Type': ct},
schema='Attachment')['json']
def wait_status(c, actor, att_id, want, seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
if att['status'] == want:
return att
if att['status'] == 'failed' and want != 'failed':
fail('attachment failed: ' + str(att.get('error_code')))
time.sleep(0.25)
fail('attachment did not reach %s in %ss' % (want, seconds))
def png_bytes(w, h, seed=0):
"""Generate a valid PNG without external libs.
Быстро и с малой памятью: одна строка пикселей переиспользуется.
Для 7000x7000 занимает ~1-2 секунды и ~150 MiB RAM.
"""
import struct, zlib
# одна строка: filter byte 0 + w * 3 байта RGB
row = bytearray(1 + w * 3)
for x in range(w):
row[1 + x*3] = (x * 7 + seed) & 0xFF
row[2 + x*3] = (x * 11) & 0xFF
row[3 + x*3] = (x * 13) & 0xFF
row = bytes(row)
# zlib потоково, без накопления всего raw
co = zlib.compressobj(level=6)
idat_parts = []
for y in range(h):
idat_parts.append(co.compress(row))
idat_parts.append(co.flush())
idat = b''.join(idat_parts)
def chunk(t, d):
return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff)
ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n'
+ chunk(b'IHDR', ihdr)
+ chunk(b'IDAT', idat)
+ chunk(b'IEND', b''))
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""kill -9 worker после S3 write до ack → повтор не создаёт дублей."""
import sys, time, signal
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = png_bytes(4000, 4000)
# Сначала готовим staging+job, но без запуска worker’а.
docker('stop', 'worker')
att = upload(c, a, ch, content)
# Запускаем worker — он возьмёт processing и, скорее всего, быстро закончит.
docker('start', 'worker')
# Ловим момент processing и убиваем процесс.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
if st == 'processing':
break
time.sleep(0.1)
docker('kill', '-s', 'SIGKILL', 'worker')
ok('worker killed during processing')
# После lease (60s) + reaper (30s) job должна снова стать queued.
time.sleep(100)
docker('start', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=120)
n = int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']))
check(n == 3, 'expected exactly 3 variants, got %d' % n)
ok('recovery produced exactly 3 non-original variants, no duplicates (original in object_key)')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Реестр failed доступен оператору (SELECT из attachment_jobs)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
att = upload(c, a, ch, content)
wait_status(c, a, att['id'], 'failed', seconds=180)
rows = psql(
"SELECT attachment_id, last_error_code, attempt FROM attachment_jobs "
"WHERE status='failed' ORDER BY updated_at DESC LIMIT 5")
check(rows, 'no failed jobs visible to operator')
ok('failed registry:\n' + rows)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Битый файл: 5 attempt → failed, error_code=invalid_image."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
# Заведомо валидный PNG в API (MIME проходит), но worker получит
# битый заголовок, если подменить staging — здесь проверяем
# реалистичный сценарий: JPEG с обрезанным хвостом.
content = (Path('tests/fixtures/sample.png')).read_bytes()[:-20]
att = upload(c, a, ch, content)
# API пропустит (DecodeConfig проходит), worker упадёт на Decode.
att = wait_status(c, a, att['id'], 'failed', seconds=180)
check(att.get('error_code') == 'invalid_image', 'unexpected error_code: %s' % att.get('error_code'))
attempts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
# invalid_image — постоянная ошибка, worker не должен ретраить.
check(attempts == 1, 'permanent error must not be retried: attempt=%d' % attempts)
ok('failed immediately (attempt=1), error_code=invalid_image — permanent, not retried')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+28
View File
@@ -0,0 +1,28 @@
"""Мини-парсер SOF для JPEG; без внешних библиотек."""
from tests.client import Failure, check
def jpeg_size(data):
check(data[:2] == b'\xff\xd8', 'thumbnail is not JPEG')
i = 2
sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf}
while i < len(data):
check(data[i] == 0xff, 'invalid JPEG marker')
while i < len(data) and data[i] == 0xff:
i += 1
check(i < len(data), 'truncated JPEG marker')
marker = data[i]
i += 1
if marker in {0xd9, 0xda}:
break
if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7:
continue
check(i + 2 <= len(data), 'truncated JPEG segment')
length = int.from_bytes(data[i:i + 2], 'big')
check(length >= 2 and i + length <= len(data), 'invalid JPEG segment length')
if marker in sof:
check(length >= 8, 'invalid JPEG SOF')
return (int.from_bytes(data[i + 5:i + 7], 'big'),
int.from_bytes(data[i + 3:i + 5], 'big'))
i += length
raise Failure('JPEG dimensions not found')
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Reaper восстанавливает job после паузы worker’а дольше lease (60с)."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = png_bytes(4000, 4000)
# 1) Готовим job, но worker не даём её взять.
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
finally:
pass # unpause ниже — после того, как настроим SIGSTOP
# 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP.
# Используем docker kill --signal=SIGSTOP, чтобы остановить процесс
# (не контейнер). Контейнер "жив", но worker заморожен.
docker('unpause', 'worker')
# 3) Ждём processing; как только взял — сразу SIGSTOP.
deadline = time.monotonic() + 15
job = None
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
if st == 'processing':
# Останавливаем процесс, пока он внутри processing.
docker('kill', '-s', 'SIGSTOP', 'worker')
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
break
if st == 'done':
fail('job finished too fast to simulate stuck worker')
time.sleep(0.02)
check(job is not None, 'worker never entered processing')
# 4) Фиксируем lease_expires_at на момент SIGSTOP.
lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job)
ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease))
# 5) Ждём, пока lease протухнет и reaper вернёт job в queued.
# lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом.
deadline = time.monotonic() + 120
requeued = False
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job)
if st == 'queued':
requeued = True
break
time.sleep(2)
check(requeued, 'reaper did not requeue stuck job (status=%s)' % st)
ok('reaper requeued the stuck job')
# 6) Возобновляем worker: он должен доделать job до ready.
docker('kill', '-s', 'SIGCONT', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=120)
ok('ready after unstuck')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""413, 415, 400 invalid_image — лимиты bytes/pixels/MIME."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import png_bytes, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
url = '/api/v1/chats/' + ch['id'] + '/attachments?filename=x.bin'
# 413: > 10 MiB
big = b'\x89PNG\r\n\x1a\n' + b'\x00' * (11 * 1024 * 1024)
r = c.call('POST', url, 413, actor=a, raw=big, headers={'Content-Type': 'image/png'})
ok('413 payload_too_large')
# 415: text/plain
r = c.call('POST', url, 415, actor=a, raw=b'hello', headers={'Content-Type': 'text/plain'})
ok('415 unsupported_media_type')
# 415: клиент говорит image/png, но содержимое не PNG/JPEG по DetectContentType
r = c.call('POST', url, 415, actor=a, raw=b'\x00' * 100, headers={'Content-Type': 'image/png'})
ok('415 mismatched content (client MIME not trusted)')
# 400: PNG-сигнатура валидна (DetectContentType -> image/png),
# но IHDR/IDAT битые -> DecodeConfig падает -> invalid_image (400)
broken_png = b'\x89PNG\r\n\x1a\n' + b'\x00' * 40
r = c.call('POST', url, 400, actor=a, raw=broken_png, headers={'Content-Type': 'image/png'})
ok('400 invalid image (signature ok, header broken)')
# 400: пиксели > MAX_IMAGE_PIXELS (если выставите 1e8 — этот тест не сработает;
# по умолчанию 40 MP, значит нужно 7000x7000 ≈ 49 MP)
huge = png_bytes(7000, 7000)
r = c.call('POST', url, 400, actor=a, raw=huge, headers={'Content-Type': 'image/png'})
ok('400 too many pixels')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Outbox: пауза outbox → 202 → unpause → ready; published_at=NULL в промежутке."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
docker('pause', 'outbox')
try:
att = upload(c, a, ch, content)
check(att['status'] == 'queued', 'expected queued')
time.sleep(3)
published = psql("SELECT coalesce(published_at::text,'NULL') FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
check(published == 'NULL', 'outbox paused but published_at=%s' % published)
ok('outbox paused: published_at IS NULL, upload accepted')
finally:
docker('unpause', 'outbox')
att = wait_status(c, a, att['id'], 'ready', seconds=90)
ok('after unpause: ready')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Повторный XADD того же job_id не создаёт дублей вариантов."""
import json, subprocess, sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'ready')
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
check(before == '3', 'expected 3 variants, got %s' % before)
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
payload = json.dumps({
"schema_version": 1, "job_id": job, "type": "attachment.process.v1",
"attachment_id": att['id'], "pipeline_version": 1, "attempt": 1,
"created_at": "2026-01-01T00:00:00Z",
})
# Повторная доставка через Redis CLI.
docker('exec', '-T', 'redis', 'redis-cli', 'XADD', 'attachment.jobs', '*', 'payload', payload)
time.sleep(5)
after = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
check(after == '3', 'redelivery created duplicate variants: %s' % after)
ok('redelivery is no-op (still 3 non-original variants; original via object_key)')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Replay failed → снова queued."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
# Форсим failed (битый файл проходит DecodeConfig, но не Decode).
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'failed', seconds=180)
ok('failed reached')
c.call('POST', '/api/v1/attachments/' + att['id'] + '/replay', 202, actor=a)
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
check(st == 'queued', 'after replay expected queued, got %s' % st)
ok('replay -> queued')
# Replay дал одну новую попытку; ошибка permanent, поэтому снова failed при attempt=1.
att = wait_status(c, a, att['id'], 'failed', seconds=180)
atts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
check(atts == 1, 'after replay: permanent error must fail at attempt=1, got %d' % atts)
ok('replay reset attempt to 1; permanent error failed again — not retried')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready."""
import subprocess, sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
# 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем.
root_user = open('secrets/minio_root_user.txt').read().strip()
root_pass = open('secrets/minio_root_password.txt').read().strip()
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc',
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
# 2) Пауза worker’а: job не будет взят до unpause.
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
check(staging.startswith('staging/'), 'unexpected staging_key')
# 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект.
# Если worker ещё не стартовал — GetObject вернёт NoSuchKey.
subprocess.run(
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm',
'loc/messenger-attachments/' + staging],
capture_output=True, text=True, check=True)
# 4) Отдаём job worker’у.
docker('unpause', 'worker')
# 5) Ждём, пока attempt >= 2.
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
deadline = time.monotonic() + 90
attempt = 1
while time.monotonic() < deadline:
attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job))
if attempt >= 2:
break
time.sleep(1)
check(attempt >= 2,
'retry did not happen on missing staging object (attempt=%d)' % attempt)
ok('transient error retried: attempt=%d' % attempt)
# 6) Логи retry именно для нашего job_id.
logs = docker('logs', '--tail=500', 'worker')
lines = [l for l in logs.splitlines()
if ('retry job=' + job) in l]
check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:]))
ok('retry log lines:')
for l in lines:
print(' ' + l)
# 7) Возвращаем staging — job должен пройти retry и стать ready.
# Перезаписываем объект с правильным содержимым.
# Через mc pipe: передаём байты на stdin.
r = subprocess.run(
['docker', 'compose', 'exec', '-T', 's3',
'mc', 'pipe', 'loc/messenger-attachments/' + staging],
input=content, capture_output=True)
check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode())
att = wait_status(c, a, att['id'], 'ready', seconds=180)
ok('recovered to ready after staging restored (attempt=%d)' % attempt)
finally:
# На всякий случай — если упали в середине, worker не должен остаться на паузе.
subprocess.run(['docker', 'compose', 'unpause', 'worker'],
capture_output=True)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""После ready staging пуст (объект перенесён в final)."""
import sys
import subprocess
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
check(staging.startswith('staging/'), 'staging_key does not start with staging/')
att = wait_status(c, a, att['id'], 'ready')
root_user = open('secrets/minio_root_user.txt').read().strip()
root_pass = open('secrets/minio_root_password.txt').read().strip()
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'local_check',
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
r = subprocess.run(
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'stat',
'local_check/messenger-attachments/' + staging],
capture_output=True, text=True)
combined = ((r.stdout or '') + (r.stderr or '')).lower()
check('does not exist' in combined or 'not found' in combined,
'staging object still present: %s\n%s' % (staging, combined))
ok('staging object removed after ready (mc rc=%d)' % r.returncode)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""queued → processing → ready через остановку worker’а."""
import time, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
check(att['status'] == 'queued', 'expected queued after upload')
ok('upload -> queued')
# Не должно перейти в processing, пока worker на паузе.
time.sleep(3)
att = c.call('GET', '/api/v1/attachments/' + att['id'], actor=a, schema='Attachment')['json']
check(att['status'] == 'queued', 'worker paused but status changed to %s' % att['status'])
ok('status stays queued while worker paused')
finally:
docker('unpause', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=90)
names = {v['name'] for v in att['variants']}
# original в variants[] нет по контракту OpenAPI — он в object_key/оригинальном файле.
check(names == {'thumbnail', 'crop', 'optimized'},
'variants mismatch: %s' % names)
ok('ready with thumbnail/crop/optimized (original available via ?variant=original)')
# original доступен отдельно — проверим, что эндпоинт отвечает 200.
r = c.call('GET',
'/api/v1/attachments/' + att['id'] + '/content?variant=original',
200, actor=a)
check(len(r['bytes']) > 0, 'original empty')
ok('original served via ?variant=original (%d bytes)' % len(r['bytes']))
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""original неизменен, thumbnail 256x256, crop 128x128, optimized существует."""
import hashlib, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, ok, fail
from student_tests.jpeg_probe import jpeg_size as dims_jpeg
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'ready')
src_sha = hashlib.sha256(content).hexdigest()
def get(v):
return c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (att['id'], v),
200, actor=a)['bytes']
orig = get('original')
check(hashlib.sha256(orig).hexdigest() == src_sha, 'original changed')
ok('original byte-identical')
th = get('thumbnail')
w, h = dims_jpeg(th)
check(max(w, h) == 256 and w <= 256 and h <= 256,
'thumbnail is %dx%d, expected fit in 256x256' % (w, h))
check(w * 160 == h * 256 if h == 160 else True, ...) # необязательно
# Проще всего — сверить с metadata:
thumb_meta = next(v for v in att['variants'] if v['name'] == 'thumbnail')
check((w, h) == (thumb_meta['width'], thumb_meta['height']),
'thumbnail metadata %sx%s != actual %dx%d' % (thumb_meta['width'], thumb_meta['height'], w, h))
ok('thumbnail %dx%d (metadata matches)' % (w, h))
cr = get('crop')
w, h = dims_jpeg(cr)
check((w, h) == (128, 128), 'crop is %dx%d' % (w, h))
ok('crop 128x128 JPEG')
opt = get('optimized')
check(len(opt) > 0, 'optimized empty')
# optimized может быть JPEG или WebP; проверяем, что не пусто и похоже на изображение
check(opt[:2] == b'\xff\xd8' or opt[:4] == b'RIFF', 'optimized is not JPEG/WebP')
ok('optimized present (%d bytes)' % len(opt))
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))