diff --git a/Dockerfile b/Dockerfile index 4ee7b168..d96d7124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # --- build stage --- -FROM golang:1.22-alpine AS build +FROM golang:1.26-alpine AS build WORKDIR /src RUN apk add --no-cache ca-certificates @@ -29,7 +29,4 @@ RUN chmod +x /app/migrate.sh USER app EXPOSE 8080 -HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \ - CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 - ENTRYPOINT ["/app/messenger"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d171..943cf6ca 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover my_tests my_tests_quick my_tests_slow help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,24 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# Быстрые (<1 мин): статусы, лимиты, варианты, payload, DLQ, staging. +my_tests_quick: + python3 student_tests/states.py + python3 student_tests/limits.py + python3 student_tests/variants.py + python3 student_tests/broker-payload.py + python3 student_tests/staging-cleanup.py + python3 student_tests/dlq.py + python3 student_tests/redelivery.py + +# Долгие (>2 мин): retry/backoff/failed/replay/outbox/lease/crash. +my_tests_slow: + python3 student_tests/outbox.py + python3 student_tests/failed.py + python3 student_tests/retry.py + python3 student_tests/replay.py + python3 student_tests/lease.py + python3 student_tests/crash-before-ack.py + +my_tests: my_tests_quick my_tests_slow \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..e32171da 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,184 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Морозова Ольга Андреевна, БВТ2303 +- Целевая оценка: **4** (треки на 5 не заявлены) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники: [README.md](README.md), [COURSE.md](COURSE.md), [contracts/openapi.json](contracts/openapi.json), документация Go, MinIO (mc), PostgreSQL, Redis Streams, Nginx, Docker. +- Библиотеки: Go 1.26 (stdlib), github.com/google/uuid v1.6.0, github.com/jackc/pgx/v5 v5.5.5, github.com/redis/go-redis/v9 v9.5.1, golang.org/x/crypto v0.23.0, github.com/minio/minio-go/v7 v7.0.72, golang.org/x/image v0.48.0. Тесты — Python 3 (stdlib). +- Помощь ИИ: Deepseek — написание и проверка кода. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Команды выполнялись в WSL2 (Ubuntu) на Windows 11. Требования: Docker 24+ (Compose v2), Python 3.9+ (для тестов), `make`. + +Версии (фиксированные теги образов): + +- `golang:1.26-alpine` — build stage +- `alpine:3.20` — runtime stage +- `postgres:16.3-alpine` — база данных +- `redis:7.2.5-alpine` — кэш сессий и брокер (Redis Streams) +- `nginx:1.27.0-alpine` — балансировщик +- `pgsty/silo:RELEASE.2026-10-26T00-00-00Z` — S3-совместимое хранилище (MinIO-compatible) + +### Подготовка + +```sh +git clone https://gitea.privcloud.tech/MiniLynx/mtusi2026lab5-fork +cd mtusi2026lab5-fork + +# 1. Секреты (не входят в Git) +mkdir -p secrets +printf 'local-dev-postgres-password\n' > secrets/postgres_password.txt +printf 'postgres://messenger:local-dev-postgres-password@postgres:5432/messenger?sslmode=disable\n' > secrets/database_url.txt +printf 'redis://redis:6379/0\n' > secrets/cache_url.txt +printf 'minioadmin\n' > secrets/minio_root_user.txt +printf 'minioadmin-dev-password\n' > secrets/minio_root_password.txt +printf 'messenger-api\n' > secrets/s3_access_key_id.txt +printf 'messenger-api-dev-secret\n' > secrets/s3_secret_access_key.txt +chmod 600 secrets/*.txt + +# 2. Локальный .env +cp .env.example .env +# ВАЖНО: В .env нужно исправить S3_ENDPOINT на s3:9000 (без http://) +sed -i 's|S3_ENDPOINT=.*|S3_ENDPOINT=s3:9000|' .env + +# 3. Python для проверок +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements-dev.txt + +# 4. Скрипты +chmod +x scripts/migrate.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh +``` + +### Сборка и запуск + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться /health/ready +docker compose ps +``` + +Для запуска с другим количеством реплик добавить `API_REPLICAS=$n` перед `docker compose --env-file .env up -d --build`, где `$n` - желаемое количество реплик. + +### Тесты + +```sh +make check +make validate PYTHON=.venv/bin/python +make test + +# Persistence +make test-persistence ACTION_SCRIPT=scripts/restart.sh + +# Failover +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +./scripts/restore-api.sh + +# Собственные тесты (pipeline, сбои, лимиты) +make my_tests +``` + +## Остановка (с сохранением volumes) + +```sh +docker compose --env-file .env down --remove-orphans +``` + +### Намеренное удаление учебных данных + +```sh +docker compose --env-file .env down --remove-orphans -v +rm -f secrets/*.txt +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP-клиент → Nginx (8080) → API (Go, x2, stateless) + │ + ├──[S3 PUT]→ staging/ (S3, приватный bucket) + ├──[INSERT]→ Postgres: attachments (queued) + attachment_jobs (outbox) + └──[202 Accepted] + +Outbox (x1) ⇄ Postgres (attachment_jobs.published_at IS NULL) + │ + └──[XADD]→ Redis Streams (attachment.jobs) + +Worker (x2) ⇄ Redis Streams (consumer group) + ├──[SELECT ... FOR UPDATE SKIP LOCKED]→ Postgres (lease 60s) + ├──[GET]→ staging/ → resize (thumb 256, crop 128, opt) →[PUT]→ final/ + ├──[INSERT variants + UPDATE ready]→ Postgres (транзакция) + └──[XACK]→ Redis +``` + +- **Асинхронный приём (уровень 3):** `POST /api/v1/chats/{id}/attachments` возвращает **202 Accepted** сразу после того, как файл сохранён в staging, а в БД одной транзакцией записаны метаданные `attachments` (status=`queued`) и `attachment_jobs` (outbox). Это гарантирует, что после ответа 202 работа не потеряется при рестарте API. +- **Outbox (уровень 4):** отдельный процесс `cmd/outbox` читает `attachment_jobs WHERE published_at IS NULL` и публикует в брокер. `published_at` ставится только после `XADD`. Падение между COMMIT и publish приводит к повторной публикации при следующем цикле; worker идемпотентен. +- **Брокер:** Redis Streams (тот же Redis, AOF `--appendonly yes`). Consumer Group `workers`, `XACK` после durable-эффекта. В сообщении только `job_id`, `attachment_id`, `pipeline_version`, `attempt` (никаких URL или filename). +- **Worker (уровень 3-4):** отдельный процесс, реплики 2. Забирает job через `ClaimJobByID` (lease 60s). Пишет `original` (server-side copy staging→final), `thumbnail` (256×256, fit), `crop` (128×128, center-cover) и `optimized` (JPEG q=70). Всё в одной транзакции с `attachment_variants` и `status='ready'`. `XACK` только после COMMIT. +- **Идемпотентность:** ключ эффекта — `(attachment_id, pipeline_version, variant)`. Повторная доставка того же job_id: `MarkReady` — no-op (если `status='done'`), `attachment_variants` — `ON CONFLICT DO NOTHING`. Также `VariantsExist` перед Claim. +- **Retry:** `RetryJob` различает permanent (`invalid_image` → сразу failed) и transient (`storage_error` → backoff `base * 2^(n-1) + jitter`, max 5 попыток). Финальный `failed` с безопасным `error_code`. +- **Зависшие job:** `RequeueStuck` в outbox и worker, каждые 30с: `status='processing' AND lease_expires_at < now()` → `queued`, `published_at=NULL`. +- **Replay:** `POST /api/v1/attachments/{id}/replay` сбрасывает `attempt=1`, `status='queued'`, `published_at=NULL` для failed-задач. +- **Очистка:** `MoveToFinal` удаляет staging после CopyObject. `ReconcileOrphans` (в API) чистит `attachment_gc` и «зависшие» staging-объекты без job. + +### Что перенесено из lab4 + +- PostgreSQL, Argon2id, sessions (с кэшем в Redis), горизонтальное масштабирование, Nginx. +- Механизм идемпотентности `idempotency_keys` для сообщений. +- S3-хранилище и приватный bucket, `S3Store` с таймаутами и ретраями. + +### Что изменено + +- Миграция `0002_attachments.sql`: добавлены `attachment_variants`, `attachment_jobs` (outbox+lease), поля `staging_key`, `pipeline_version`, `ready_at`, `failed_at`. Таблица `attachments` теперь содержит `status IN ('queued','processing','ready','failed')`. +- `UploadAttachment` возвращает 202, статус `queued`; `DownloadAttachment` → 409 для не-ready. +- Добавлены `cmd/outbox`, `cmd/worker` (обработка изображений, `process.go`). +- `src/domain/broker.go` — обёртка Redis Streams. +- `compose.yaml`: сервисы `outbox` (1 реплика) и `worker` (2 реплики), миграции вынесены в отдельный сервис `migrate`. +- В `attachments.go`: `EnqueueAttachment`, `ClaimJob`, `ClaimJobByID`, `MarkReady`, `RetryJob`, `RequeueStuck`, `ReplayFailed`, `MarkFailedPermanent`, `GetVariantKey`, `SelectUnpublishedJobs`, `MarkPublished`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------ | +| Offline-структура и локальные refs | `make check` | PASS | [png](evidence/2_Tests_Passing.png) | +| Полная OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | [png](evidence/2_Tests_Passing.png) | +| Публичный smoke (lab5) | `make test` | PASS: 202, ready, thumbnail | [png](evidence/2_Tests_Passing.png) | +| Persistence: перезапуск API+БД+Redis+S3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | [png](evidence/2_Tests_Passing.png) | +| Failover: остановка одной реплики API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | `observed_instances_before: 2`, `successful_writes: 75`, `first_recovery: 5.371s` | [png](evidence/2_Tests_Passing.png) | +| **Ур. 3:** Upload → 202 → queued (worker не нужен) | `student_tests/states.py` | queued → ready, 3 варианта + original через `?variant=original` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Broker и worker — отдельные процессы | `docker compose ps` | `outbox`, `worker-1`, `worker-2` | [png](evidence/1_Healthcheck.png) | +| **Ур. 3:** Статусы в API, лимиты и типы | `student_tests/limits.py` | 413, 415, 400 invalid_image | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** original неизменен, thumbnail по контракту | `student_tests/variants.py` | `byte-identical`, `thumbnail 256x160 (metadata matches)`, `crop 128x128` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Broker payload без filename/URL | `student_tests/broker-payload.py` | только id/version/attempt | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Очистка staging после ready | `student_tests/staging-cleanup.py` | `staging object removed after ready` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Реестр failed (DLQ) | `student_tests/dlq.py` | `invalid_image\|1` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 3:** Идемпотентность при повторной доставке | `student_tests/redelivery.py` | `still 3 non-original variants` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Outbox закрывает разрыв COMMIT→publish | `student_tests/outbox.py` | `published_at IS NULL` при паузе outbox → ready после unpause | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Permanent error без retry | `student_tests/failed.py` | `attempt=1, invalid_image — not retried` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Transient error → retry → ready | `student_tests/retry.py` | `attempt=2`, `retry log lines: ... backoff=2s jitter=451ms` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Replay failed | `student_tests/replay.py` | `replay -> queued`, повторный permanent fail | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Lease / reaper зависших | `student_tests/lease.py` | `reaper requeued the stuck job` → `ready` | [txt](evidence/my_tests_passing.txt) | +| **Ур. 4:** Crash после side effect до ack | `student_tests/crash-before-ack.py` | `recovery produced exactly 3 variants, no duplicates` | [txt](evidence/my_tests_passing.txt) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- | +| Остановка одной реплики API | 0 потерянных подтверждённых записей, LB переключается | Failover PASS, 75 успешных сообщений | 5.371 с, потерь нет | +| Рестарт API/БД/Redis/S3 (volumes) | Вложение и сессия живы | Persistence PASS | ~30 с (запуск контейнеров), потерь нет | +| Отказ S3 | API возвращает 503, worker ретраит | `retry.py` PASS, `attempt=2`, backoff сработал | После восстановления S3 — ready | +| Пауза outbox (разрыв COMMIT→publish) | `published_at=NULL`, upload 202 | `outbox.py` PASS, job дошёл до ready после unpause | Задержка publish = период паузы, потерь нет | +| Пауза worker (зависший processing) | Lease истекает, reaper возвращает в queued | `lease.py` PASS, job вернулся в queued через ~60с + reaper | ~90 с, потерь нет | +| `SIGKILL` worker во время processing | Повторная доставка, без дублей вариантов | `crash-before-ack.py` PASS, 3 варианта, без дублей | ~100 с (lease + рестарт), потерь нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **At-least-once:** доставка Redis Streams + идемпотентный эффект ≠ exactly-once. При падении между XADD и MarkPublished возможна повторная публикация; повторный worker это корректно обрабатывает, но сообщение в Stream продублируется. +- **Replay** доступен через endpoint `POST /api/v1/attachments/{id}/replay` (расширение сверх OpenAPI). В отчёте зафиксировано, что это не меняет семантику обязательных путей. +- **Optimized** — JPEG q=70 вместо WebP (WebP-кодирование требует cgo-зависимостей, которых в alpine-сборке без CGO нет). Контракт допускает JPEG как «перекодированный» вариант; в отчёте явно указано. +- **Broker = Redis Streams** вместо RabbitMQ — обосновано тем, что Redis уже нужен для кэша сессий, AOF включён, это уменьшает количество контейнеров. Если Redis упадёт без AOF-реплики, неопубликованные задачи будут ждать в `attachment_jobs` (outbox) — они не потеряются. +- **DLQ** — пока реестр `attachment_jobs.status='failed'` + `last_error_code`, без отдельного UI. Для оператора — `SELECT` из БД и endpoint replay. +- **Не реализовано:** версионирование нескольких pipeline (трек 5), auto-scaling worker’а по queue lag (трек 1), fault-injection с версионированием (трек 2). Уровень 5 не заявлен. +- **Нагрузка** проверена только smoke + `make test`; полноценных замеров throughput/latency pipeline не делалось (не требуется для уровня 4). diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 25e961d6..f16389b3 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -168,7 +168,8 @@ func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store, return } } else { - sum, err := s3.PutBytes(ctx, finalOriginalKey, v.ContentType, v.Data) + key := "final/" + j.AttachmentID + "/" + v.Name + sum, err := s3.PutBytes(ctx, key, v.ContentType, v.Data) if err != nil { _, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff) _ = broker.Ack(ctx, m.ID) diff --git a/cmd/worker/process.go b/cmd/worker/process.go index 58a05293..eb7500ea 100644 --- a/cmd/worker/process.go +++ b/cmd/worker/process.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "image" "image/jpeg" + "math" "golang.org/x/image/draw" ) @@ -29,7 +30,7 @@ func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) { // processImage возвращает 4 варианта. Оригинал — byte-identical с входом. // thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop), -// optimized — WebP q=80 без метаданных. +// optimized — JPEG q=70 без метаданных (WebP оставлен на усмотрение; см. отчёт). func processImage(raw []byte, contentType string) ([]VariantSpec, error) { src, _, err := image.Decode(bytes.NewReader(raw)) if err != nil { @@ -46,14 +47,14 @@ func processImage(raw []byte, contentType string) ([]VariantSpec, error) { Data: raw, SHA256: hex.EncodeToString(sum[:]), }) - // thumbnail 256x256 (fit). + // thumbnail: fit в 256x160 с сохранением пропорций (для 16:10 — 256x160). th := resizeFit(src, 256, 256) if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil { return nil, err } - // crop 128x128 (center crop). - cr := centerCrop(src, 128, 128) + // crop 128x128: центр-cover до меньшего квадрата. + cr := squareCover(src, 128) if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil { return nil, err } @@ -96,28 +97,41 @@ func resizeFit(src image.Image, maxW, maxH int) image.Image { if scale > 1 { scale = 1 } - nw, nh := int(float64(w)*scale), int(float64(h)*scale) + nw := int(math.Round(float64(w) * scale)) + nh := int(math.Round(float64(h) * scale)) dst := image.NewRGBA(image.Rect(0, 0, nw, nh)) draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil) return dst } -func centerCrop(src image.Image, w, h int) image.Image { +// squareCover масштабирует src так, чтобы покрыть квадрат size×size +// (min-масштаб по двум осям), затем центр-кроп до size×size. +// Результат — всегда ровно size×size. +func squareCover(src image.Image, size int) image.Image { b := src.Bounds() - sw, sh := b.Dx(), b.Dy() - cw, ch := w, h - if sw < cw { - cw = sw + w, h := b.Dx(), b.Dy() + // scale — больший из двух, чтобы покрыть квадрат + scale := float64(size) / float64(w) + if float64(size)/float64(h) > scale { + scale = float64(size) / float64(h) } - if sh < ch { - ch = sh + nw := int(math.Round(float64(w) * scale)) + nh := int(math.Round(float64(h) * scale)) + if nw < size { + nw = size } - x0 := (sw - cw) / 2 - y0 := (sh - ch) / 2 - dst := image.NewRGBA(image.Rect(0, 0, w, h)) - // масштабируем crop-область в целевой размер (иначе мелкие картинки не влезут). - draw.CatmullRom.Scale(dst, dst.Bounds(), - src, image.Rect(b.Min.X+x0, b.Min.Y+y0, b.Min.X+x0+cw, b.Min.Y+y0+ch), - draw.Over, nil) + if nh < size { + nh = size + } + + scaled := image.NewRGBA(image.Rect(0, 0, nw, nh)) + draw.CatmullRom.Scale(scaled, scaled.Bounds(), src, b, draw.Over, nil) + + // center-crop до size×size + x0 := (nw - size) / 2 + y0 := (nh - size) / 2 + dst := image.NewRGBA(image.Rect(0, 0, size, size)) + draw.Draw(dst, dst.Bounds(), scaled, + image.Pt(x0, y0), draw.Src) return dst } diff --git a/compose.yaml b/compose.yaml index 4b4939a1..3614cc85 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,6 +10,7 @@ x-api: &api redis: { condition: service_healthy } s3: { condition: service_healthy } s3-init: { condition: service_completed_successfully } + migrate: { condition: service_completed_successfully } networks: [backend] services: @@ -91,6 +92,12 @@ services: limits: cpus: "${API_CPU_PER_REPLICA:-0.5}" memory: "${API_MEM_PER_REPLICA:-128M}" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 10s + timeout: 3s + start_period: 20s + retries: 5 outbox: <<: *api @@ -98,6 +105,7 @@ services: depends_on: postgres: { condition: service_healthy } redis: { condition: service_healthy } + migrate: { condition: service_completed_successfully } deploy: replicas: 1 resources: @@ -110,6 +118,7 @@ services: postgres: { condition: service_healthy } redis: { condition: service_healthy } s3: { condition: service_healthy } + migrate: { condition: service_completed_successfully } deploy: replicas: 2 resources: diff --git a/evidence/1_Healthcheck.png b/evidence/1_Healthcheck.png new file mode 100644 index 00000000..951815e3 Binary files /dev/null and b/evidence/1_Healthcheck.png differ diff --git a/evidence/2_Tests_Passing.png b/evidence/2_Tests_Passing.png new file mode 100644 index 00000000..d1e3ffb2 Binary files /dev/null and b/evidence/2_Tests_Passing.png differ diff --git a/evidence/my_tests_passing.txt b/evidence/my_tests_passing.txt new file mode 100644 index 00000000..1159c9d8 --- /dev/null +++ b/evidence/my_tests_passing.txt @@ -0,0 +1,56 @@ +(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ make my_tests +python3 student_tests/states.py +upload -> queued +status stays queued while worker paused +ready with thumbnail/crop/optimized (original available via ?variant=original) +original served via ?variant=original (111817 bytes) +python3 student_tests/limits.py +413 payload_too_large +415 unsupported_media_type +415 mismatched content (client MIME not trusted) +400 invalid image (signature ok, header broken) +400 too many pixels +python3 student_tests/variants.py +original byte-identical +thumbnail 256x160 (metadata matches) +crop 128x128 JPEG +optimized present (4818 bytes) +python3 student_tests/broker-payload.py +stream sample: +1794848167433-0 +payload +{"schema_version":1,"job_id":"ee59551f-c85f-411a-b4d9-6076f31fde8f","type":"attachment.process.v1","attachment_id":"1175cf24-2984-4800-b84d-1aa702ecd7bd","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:07Z"} +1794848176445-0 +payload +{"schema_version":1,"job_id":"b7792daf-6123-4c7a-9566-b2fc4a53a614","type":"attachment.process.v1","attachment_id":"05ef10e7-19c4-47ef-9c68-a2e09eb8338e","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:16Z"} + +broker payload has only ids/version/attempt +python3 student_tests/staging-cleanup.py +staging object removed after ready (mc rc=1) +python3 student_tests/dlq.py +failed registry: +b1ccbeca-ae22-44f1-837c-868d2165889c|invalid_image|1 +python3 student_tests/redelivery.py +redelivery is no-op (still 3 non-original variants; original via object_key) +python3 student_tests/outbox.py +outbox paused: published_at IS NULL, upload accepted +after unpause: ready +python3 student_tests/failed.py +failed immediately (attempt=1), error_code=invalid_image — permanent, not retried +python3 student_tests/retry.py +transient error retried: attempt=2 +retry log lines: + worker-1 | 2026/11/16 16:56:48 retry job=64072f3d-6b9e-4101-b31b-7e9a2f51bac3 attempt=2 next_attempt=2026-11-16T16:56:50.964432673Z backoff=2s jitter=451.35473ms code=storage_error +recovered to ready after staging restored (attempt=2) +python3 student_tests/replay.py +failed reached +replay -> queued +replay reset attempt to 1; permanent error failed again — not retried +python3 student_tests/lease.py +worker stopped at processing (job=da0adcf7-64a7-427c-a50b-438e9d688bb7, lease_expires_at=2026-11-16 16:57:59.829609+00) +reaper requeued the stuck job +ready after unstuck +python3 student_tests/crash-before-ack.py +worker killed during processing +recovery produced exactly 3 non-original variants, no duplicates (original in object_key) +(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ \ No newline at end of file diff --git a/migrations/0001_core.sql b/migrations/0001_core.sql new file mode 100644 index 00000000..3dd32330 --- /dev/null +++ b/migrations/0001_core.sql @@ -0,0 +1,58 @@ +-- ============================================================ +-- Core: users, sessions, chats, members, messages, idempotency +-- ============================================================ + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id); +CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор. +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys (created_at); \ No newline at end of file diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql deleted file mode 100644 index 3925e4ac..00000000 --- a/migrations/0001_init.sql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE TABLE IF NOT EXISTS users ( - id UUID PRIMARY KEY, - username TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - password_hash TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS chats ( - id UUID PRIMARY KEY, - title TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS chat_members ( - chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - PRIMARY KEY (chat_id, user_id) -); - -CREATE TABLE IF NOT EXISTS messages ( - id UUID PRIMARY KEY, - chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, - sender_id UUID NOT NULL REFERENCES users(id), - text TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - --- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор. -CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id - ON messages (chat_id, created_at, id); \ No newline at end of file diff --git a/migrations/0002_attachments.sql b/migrations/0002_attachments.sql new file mode 100644 index 00000000..d3861270 --- /dev/null +++ b/migrations/0002_attachments.sql @@ -0,0 +1,88 @@ +-- ============================================================ +-- Attachments: metadata, variants, jobs/outbox, GC +-- ============================================================ + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + object_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0 AND size_bytes <= 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width INTEGER NOT NULL CHECK (width >= 0), + height INTEGER NOT NULL CHECK (height >= 0), + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued','processing','ready','failed','pending')), + error_code TEXT, + -- поля из 0006 + pipeline_version INTEGER NOT NULL DEFAULT 1, + staging_key TEXT, + ready_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id); +-- Для reconciler/поиска «зависших» pending-записей: +CREATE INDEX IF NOT EXISTS idx_attachments_pending + ON attachments (created_at) WHERE status <> 'ready'; + +-- Связь сообщение ↔ вложения. FK на элементы массива Postgres не умеет, +-- целостность — на уровне приложения (или триггера, если нужно). +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}'; + +-- Журнал компенсации: объекты S3, которые могли остаться без метаданных. +CREATE TABLE IF NOT EXISTS attachment_gc ( + object_key TEXT PRIMARY KEY, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Idempotency эффекта: (attachment_id, pipeline_version, variant) +-- content_type без CHECK: варианты могут быть webp и т.п. +-- sha256 — text+CHECK, чтобы не ловить padding от char(64). +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + variant TEXT NOT NULL, + object_key TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +-- Outbox + очередь job-ов +CREATE TABLE IF NOT EXISTS attachment_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued','processing','done','failed')), + -- outbox + published_at TIMESTAMPTZ, + publish_attempts INTEGER NOT NULL DEFAULT 0, + -- lease + lease_owner TEXT, + lease_expires_at TIMESTAMPTZ, + -- retry + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию +); +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished + ON attachment_jobs (next_attempt_at) + WHERE published_at IS NULL AND status = 'queued'; +CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease + ON attachment_jobs (lease_expires_at) + WHERE status = 'processing'; \ No newline at end of file diff --git a/migrations/0002_sessions.sql b/migrations/0002_sessions.sql deleted file mode 100644 index f77b30d7..00000000 --- a/migrations/0002_sessions.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE IF NOT EXISTS sessions ( - token TEXT PRIMARY KEY, - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - expires_at TIMESTAMPTZ NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id); -CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at); \ No newline at end of file diff --git a/migrations/0003_messages_alt_index.sql b/migrations/0003_messages_covering_optional.sql similarity index 70% rename from migrations/0003_messages_alt_index.sql rename to migrations/0003_messages_covering_optional.sql index 12d58f11..dda6537d 100644 --- a/migrations/0003_messages_alt_index.sql +++ b/migrations/0003_messages_covering_optional.sql @@ -1,5 +1,5 @@ -- Индекс B для «стоимости записи»: -- покрывающий индекс, включающий text — растёт быстрее, помогает только --- покрывающим запросам. +-- покрывающим запросам. Накатывать отдельно, чтобы сравнить с индексом A. CREATE INDEX IF NOT EXISTS idx_messages_chat_covering ON messages (chat_id, created_at, id) INCLUDE (sender_id, text); \ No newline at end of file diff --git a/migrations/0004_idempotency.sql b/migrations/0004_idempotency.sql deleted file mode 100644 index da621207..00000000 --- a/migrations/0004_idempotency.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE IF NOT EXISTS idempotency_keys ( - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - method TEXT NOT NULL, - path TEXT NOT NULL, - key TEXT NOT NULL, - request_hash TEXT NOT NULL, - status_code INTEGER NOT NULL, - response JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, method, path, key) -); - -CREATE INDEX IF NOT EXISTS idx_idempotency_created - ON idempotency_keys (created_at); \ No newline at end of file diff --git a/migrations/0005_attachments.sql b/migrations/0005_attachments.sql deleted file mode 100644 index d8765167..00000000 --- a/migrations/0005_attachments.sql +++ /dev/null @@ -1,30 +0,0 @@ -CREATE TABLE IF NOT EXISTS attachments ( - id UUID PRIMARY KEY, - chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, - owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - object_key TEXT NOT NULL UNIQUE, - filename TEXT NOT NULL, - content_type TEXT NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), - size_bytes BIGINT NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760), - sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), - width INTEGER NOT NULL CHECK (width > 0), - height INTEGER NOT NULL CHECK (height > 0), - status TEXT NOT NULL DEFAULT 'ready' CHECK (status IN ('pending','ready','failed')), - error_code TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id); -CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id); --- Для reconciler/поиска «зависших» pending-записей: -CREATE INDEX IF NOT EXISTS idx_attachments_pending ON attachments (created_at) WHERE status <> 'ready'; - -ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}'; - --- Журнал компенсации: объекты S3, которые могли остаться без метаданных. -CREATE TABLE IF NOT EXISTS attachment_gc ( - object_key TEXT PRIMARY KEY, - reason TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); \ No newline at end of file diff --git a/migrations/0006_attachment_jobs.sql b/migrations/0006_attachment_jobs.sql deleted file mode 100644 index bbf60e53..00000000 --- a/migrations/0006_attachment_jobs.sql +++ /dev/null @@ -1,49 +0,0 @@ --- staging → final prefix; job-очередь и lease -ALTER TABLE attachments - ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1, - ADD COLUMN IF NOT EXISTS staging_key text, - ADD COLUMN IF NOT EXISTS ready_at timestamptz, - ADD COLUMN IF NOT EXISTS failed_at timestamptz, - ADD COLUMN IF NOT EXISTS error_code text; - --- idempotency эффекта: (attachment_id, pipeline_version, variant) -CREATE TABLE IF NOT EXISTS attachment_variants ( - attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, - pipeline_version integer NOT NULL, - variant text NOT NULL, - object_key text NOT NULL, - content_type text NOT NULL, - size_bytes bigint NOT NULL, - width int NOT NULL, - height int NOT NULL, - sha256 char(64) NOT NULL, - created_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (attachment_id, pipeline_version, variant) -); - --- outbox + очередь job-ов -CREATE TABLE IF NOT EXISTS attachment_jobs ( - job_id uuid PRIMARY KEY, - attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, - pipeline_version integer NOT NULL, - attempt int NOT NULL DEFAULT 1, - status text NOT NULL DEFAULT 'queued', -- queued|processing|done|failed - -- outbox - published_at timestamptz, - publish_attempts int NOT NULL DEFAULT 0, - -- lease - lease_owner text, - lease_expires_at timestamptz, - -- retry - next_attempt_at timestamptz NOT NULL DEFAULT now(), - last_error_code text, - created_at timestamptz NOT NULL DEFAULT now(), - updated_at timestamptz NOT NULL DEFAULT now(), - UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию -); -CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished - ON attachment_jobs (next_attempt_at) - WHERE published_at IS NULL AND status = 'queued'; -CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease - ON attachment_jobs (lease_expires_at) - WHERE status = 'processing'; \ No newline at end of file diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 5d3457a3..4f3d4864 100644 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -6,8 +6,20 @@ URL_FILE="${DATABASE_URL_FILE:-/run/secrets/database_url}" DATABASE_URL="$(cat "$URL_FILE")" PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1 -q" -for f in migrations/*.sql; do - echo "applying $f" - $PSQL -f "$f" +echo "waiting for postgres..." +for i in $(seq 1 60); do + if psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null 2>&1; then + echo "postgres is ready" + break + fi + echo " attempt $i: not ready yet" + sleep 1 +done + +psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null + +for f in /app/migrations/*.sql; do + echo "applying $f" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$f" done echo "migrations done" \ No newline at end of file diff --git a/src/api/handlers.go b/src/api/handlers.go index 640f47b9..4b63ec97 100644 --- a/src/api/handlers.go +++ b/src/api/handlers.go @@ -459,7 +459,14 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { pendingID, err := h.Store.CreateAttachmentPending( r.Context(), chatID, me.ID, stagingKey, filename) if err != nil { - writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + switch { + case errors.Is(err, domain.ErrInvalid): + writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid attachment metadata") + case errors.Is(err, domain.ErrTimeout): + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout") + default: + writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable") + } return } diff --git a/src/domain/attachment_states.go b/src/domain/attachment_states.go index 17189f6e..15e27aa8 100644 --- a/src/domain/attachment_states.go +++ b/src/domain/attachment_states.go @@ -9,4 +9,4 @@ const ( const PipelineVersion = 1 -var RequiredVariants = []string{"original", "thumbnail", "crop", "optimized"} +var RequiredVariants = []string{"thumbnail", "crop", "optimized"} diff --git a/src/domain/attachments.go b/src/domain/attachments.go index c3094ec4..dd9e8137 100644 --- a/src/domain/attachments.go +++ b/src/domain/attachments.go @@ -3,6 +3,7 @@ package domain import ( "context" "errors" + "log" "math/rand" "time" @@ -48,12 +49,12 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A var errCode *string err := s.pool.QueryRow(ctx, ` SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, - a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version - FROM attachments a - JOIN chat_members m ON m.chat_id = a.chat_id - WHERE a.id=$1 AND m.user_id=$2`, id, userID, + a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id + WHERE a.id=$1 AND m.user_id=$2`, id, userID, ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, - &a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt) + &a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNotFound } @@ -64,10 +65,11 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A a.Variants = []Variant{} rows, err := s.pool.Query(ctx, `SELECT variant, content_type, size_bytes, width, height, sha256, object_key - FROM attachment_variants - WHERE attachment_id=$1 AND pipeline_version=$2 - ORDER BY variant`, - a.ID, a.PipelineVersion) + FROM attachment_variants + WHERE attachment_id=$1 AND pipeline_version=$2 + AND variant = ANY($3) + ORDER BY variant`, + a.ID, a.PipelineVersion, RequiredVariants) if err != nil { return nil, classifyDBError(err) } @@ -111,10 +113,13 @@ func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string, return ErrInvalid } seen[id] = true - var ok bool + var status string err := s.pool.QueryRow(ctx, - `SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`, - id, chatID).Scan(&ok) + `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`, + id, chatID).Scan(&status) + if err == nil && status == AttachmentFailed { + return ErrInvalid + } if errors.Is(err, pgx.ErrNoRows) { return ErrNotFound } @@ -199,10 +204,10 @@ func (s *Store) EnqueueAttachment( pipeline_version=$7, status='queued', updated_at=now() WHERE id=$1 AND status='queued' RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - status, error_code, created_at, updated_at`, + status, error_code, created_at, updated_at, pipeline_version`, attachmentID, contentType, size, sha, w, h, pipelineVersion, ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, - &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt) + &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNotFound } @@ -312,10 +317,13 @@ func (s *Store) MarkReady( } for _, v := range variants { + if v.Name == "original" { + continue // original живёт в attachments.object_key, не в variants + } if _, err := tx.Exec(ctx, ` INSERT INTO attachment_variants - (attachment_id, pipeline_version, variant, object_key, - content_type, size_bytes, width, height, sha256) + (attachment_id, pipeline_version, variant, object_key, + content_type, size_bytes, width, height, sha256) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`, attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name, @@ -374,6 +382,8 @@ func (s *Store) RetryJob( if err := tx.Commit(ctx); err != nil { return false, classifyDBError(err) } + log.Printf("retry job=%s exhausted attempts=%d code=%s", + jobID, attempt, errorCode) return true, nil } @@ -388,6 +398,7 @@ func (s *Store) RetryJob( status = 'queued', last_error_code=$2, next_attempt_at=$3, + published_at=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now() WHERE job_id=$1`, jobID, errorCode, next); err != nil { return false, classifyDBError(err) @@ -395,16 +406,21 @@ func (s *Store) RetryJob( if err := tx.Commit(ctx); err != nil { return false, classifyDBError(err) } + // Логируем зафиксированный retry — используется в student-tests/backoff.py + log.Printf("retry job=%s attempt=%d next_attempt=%s backoff=%s jitter=%s code=%s", + jobID, attempt+1, next.UTC().Format(time.RFC3339Nano), + backoff, jitter, errorCode) + return false, nil } // RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox. func (s *Store) RequeueStuck(ctx context.Context) (int64, error) { tag, err := s.pool.Exec(ctx, ` - UPDATE attachment_jobs - SET status='queued', lease_owner=NULL, lease_expires_at=NULL, - next_attempt_at = now(), updated_at = now() - WHERE status='processing' AND lease_expires_at < now()`) + UPDATE attachment_jobs + SET status='queued', lease_owner=NULL, lease_expires_at=NULL, + next_attempt_at = now(), published_at = NULL, updated_at = now() + WHERE status='processing' AND lease_expires_at < now()`) if err != nil { return 0, classifyDBError(err) } @@ -421,9 +437,10 @@ func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error { tag, err := tx.Exec(ctx, ` UPDATE attachment_jobs - SET status='queued', attempt=1, next_attempt_at=now(), - last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now() - WHERE attachment_id=$1 AND status='failed'`, attachmentID) + SET status='queued', attempt=1, next_attempt_at=now(), + published_at=NULL, + last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now() + WHERE attachment_id=$1 AND status='failed'`, attachmentID) if err != nil { return classifyDBError(err) } diff --git a/src/domain/postgres.go b/src/domain/postgres.go index 3497f4c6..a8689d1e 100644 --- a/src/domain/postgres.go +++ b/src/domain/postgres.go @@ -526,15 +526,26 @@ func (s *Store) AddMessageIdempotent( return nil, nil, false, ErrInvalid } seen[aid] = true - var ok bool + var ( + status string + objectKey string + ) + // Вложение из этого же чата; не обязательно ready — клиент увидит картинку, + // когда worker завершит обработку. failed — принимаем: сообщение уже создано, + // UI покажет «failed» через GET /attachments/{id}. if err := tx.QueryRow(ctx, - `SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`, - aid, chatID).Scan(&ok); err != nil { + `SELECT status, object_key FROM attachments WHERE id=$1 AND chat_id=$2`, + aid, chatID).Scan(&status, &objectKey); err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, nil, false, ErrNotFound } return nil, nil, false, classifyDBError(err) } + if status == AttachmentFailed { + // Нельзя прикреплять заведомо неудачные вложения — иначе UI будет + // навсегда показывать сломанную картинку без пути к replay. + return nil, nil, false, ErrInvalid + } } // 2) Уже был такой ключ? diff --git a/student_tests/broker-payload.py b/student_tests/broker-payload.py new file mode 100644 index 00000000..d552066f --- /dev/null +++ b/student_tests/broker-payload.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""В Stream нет filename/URL/секретов — только id/version/attempt.""" +import json, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + upload(c, a, ch, content) + raw = docker('exec', '-T', 'redis', 'redis-cli', '--raw', 'XRANGE', 'attachment.jobs', '-', '+') + ok('stream sample:\n' + raw) + lower = raw.lower() + for bad in ('filename', 'sample.png', 'staging/', 'final/', 'http', 'secret'): + check(bad not in lower, 'broker payload contains forbidden token: %s' % bad) + ok('broker payload has only ids/version/attempt') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 00000000..4732d3f2 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,73 @@ +"""Shared helpers for student lab5 tests; no tokens are logged.""" +import base64, json, os, subprocess, sys, time, uuid +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, Failure, check # noqa: E402 + +def fail(msg): + print('FAIL:', msg) + raise SystemExit(1) + +def ok(msg='PASS'): + print(msg) + +def docker(*args, check_rc=True): + """Run docker compose ; return stdout. Raises if rc!=0 and check_rc.""" + rc = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True) + if check_rc and rc.returncode != 0: + raise Failure('docker compose %s: %s' % (' '.join(args), rc.stderr.strip())) + return rc.stdout + +def psql(sql): + return docker('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-tAc', sql).strip() + +def upload(c, actor, chat, content, filename='sample.png', ct='image/png'): + return c.call('POST', + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, + 202, actor=actor, raw=content, headers={'Content-Type': ct}, + schema='Attachment')['json'] + +def wait_status(c, actor, att_id, want, seconds=60): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] == want: + return att + if att['status'] == 'failed' and want != 'failed': + fail('attachment failed: ' + str(att.get('error_code'))) + time.sleep(0.25) + fail('attachment did not reach %s in %ss' % (want, seconds)) + +def png_bytes(w, h, seed=0): + """Generate a valid PNG without external libs. + + Быстро и с малой памятью: одна строка пикселей переиспользуется. + Для 7000x7000 занимает ~1-2 секунды и ~150 MiB RAM. + """ + import struct, zlib + + # одна строка: filter byte 0 + w * 3 байта RGB + row = bytearray(1 + w * 3) + for x in range(w): + row[1 + x*3] = (x * 7 + seed) & 0xFF + row[2 + x*3] = (x * 11) & 0xFF + row[3 + x*3] = (x * 13) & 0xFF + row = bytes(row) + + # zlib потоково, без накопления всего raw + co = zlib.compressobj(level=6) + idat_parts = [] + for y in range(h): + idat_parts.append(co.compress(row)) + idat_parts.append(co.flush()) + idat = b''.join(idat_parts) + + def chunk(t, d): + return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff) + + ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', ihdr) + + chunk(b'IDAT', idat) + + chunk(b'IEND', b'')) \ No newline at end of file diff --git a/student_tests/crash-before-ack.py b/student_tests/crash-before-ack.py new file mode 100644 index 00000000..76e8da65 --- /dev/null +++ b/student_tests/crash-before-ack.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""kill -9 worker после S3 write до ack → повтор не создаёт дублей.""" +import sys, time, signal +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = png_bytes(4000, 4000) + + # Сначала готовим staging+job, но без запуска worker’а. + docker('stop', 'worker') + att = upload(c, a, ch, content) + # Запускаем worker — он возьмёт processing и, скорее всего, быстро закончит. + docker('start', 'worker') + # Ловим момент processing и убиваем процесс. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + if st == 'processing': + break + time.sleep(0.1) + docker('kill', '-s', 'SIGKILL', 'worker') + ok('worker killed during processing') + + # После lease (60s) + reaper (30s) job должна снова стать queued. + time.sleep(100) + docker('start', 'worker') + att = wait_status(c, a, att['id'], 'ready', seconds=120) + n = int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])) + check(n == 3, 'expected exactly 3 variants, got %d' % n) + ok('recovery produced exactly 3 non-original variants, no duplicates (original in object_key)') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/dlq.py b/student_tests/dlq.py new file mode 100644 index 00000000..1e90520b --- /dev/null +++ b/student_tests/dlq.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Реестр failed доступен оператору (SELECT из attachment_jobs).""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes()[:-20] + att = upload(c, a, ch, content) + wait_status(c, a, att['id'], 'failed', seconds=180) + rows = psql( + "SELECT attachment_id, last_error_code, attempt FROM attachment_jobs " + "WHERE status='failed' ORDER BY updated_at DESC LIMIT 5") + check(rows, 'no failed jobs visible to operator') + ok('failed registry:\n' + rows) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/failed.py b/student_tests/failed.py new file mode 100644 index 00000000..47aa8daa --- /dev/null +++ b/student_tests/failed.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Битый файл: 5 attempt → failed, error_code=invalid_image.""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + # Заведомо валидный PNG в API (MIME проходит), но worker получит + # битый заголовок, если подменить staging — здесь проверяем + # реалистичный сценарий: JPEG с обрезанным хвостом. + content = (Path('tests/fixtures/sample.png')).read_bytes()[:-20] + att = upload(c, a, ch, content) + # API пропустит (DecodeConfig проходит), worker упадёт на Decode. + att = wait_status(c, a, att['id'], 'failed', seconds=180) + check(att.get('error_code') == 'invalid_image', 'unexpected error_code: %s' % att.get('error_code')) + attempts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])) + # invalid_image — постоянная ошибка, worker не должен ретраить. + check(attempts == 1, 'permanent error must not be retried: attempt=%d' % attempts) + ok('failed immediately (attempt=1), error_code=invalid_image — permanent, not retried') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/jpeg_probe.py b/student_tests/jpeg_probe.py new file mode 100644 index 00000000..1ea8051f --- /dev/null +++ b/student_tests/jpeg_probe.py @@ -0,0 +1,28 @@ +"""Мини-парсер SOF для JPEG; без внешних библиотек.""" +from tests.client import Failure, check + + +def jpeg_size(data): + check(data[:2] == b'\xff\xd8', 'thumbnail is not JPEG') + i = 2 + sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf} + while i < len(data): + check(data[i] == 0xff, 'invalid JPEG marker') + while i < len(data) and data[i] == 0xff: + i += 1 + check(i < len(data), 'truncated JPEG marker') + marker = data[i] + i += 1 + if marker in {0xd9, 0xda}: + break + if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7: + continue + check(i + 2 <= len(data), 'truncated JPEG segment') + length = int.from_bytes(data[i:i + 2], 'big') + check(length >= 2 and i + length <= len(data), 'invalid JPEG segment length') + if marker in sof: + check(length >= 8, 'invalid JPEG SOF') + return (int.from_bytes(data[i + 5:i + 7], 'big'), + int.from_bytes(data[i + 3:i + 5], 'big')) + i += length + raise Failure('JPEG dimensions not found') \ No newline at end of file diff --git a/student_tests/lease.py b/student_tests/lease.py new file mode 100644 index 00000000..e77c5c50 --- /dev/null +++ b/student_tests/lease.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Reaper восстанавливает job после паузы worker’а дольше lease (60с).""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = png_bytes(4000, 4000) + + # 1) Готовим job, но worker не даём её взять. + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + finally: + pass # unpause ниже — после того, как настроим SIGSTOP + + # 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP. + # Используем docker kill --signal=SIGSTOP, чтобы остановить процесс + # (не контейнер). Контейнер "жив", но worker заморожен. + docker('unpause', 'worker') + + # 3) Ждём processing; как только взял — сразу SIGSTOP. + deadline = time.monotonic() + 15 + job = None + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + if st == 'processing': + # Останавливаем процесс, пока он внутри processing. + docker('kill', '-s', 'SIGSTOP', 'worker') + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + break + if st == 'done': + fail('job finished too fast to simulate stuck worker') + time.sleep(0.02) + + check(job is not None, 'worker never entered processing') + + # 4) Фиксируем lease_expires_at на момент SIGSTOP. + lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job) + ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease)) + + # 5) Ждём, пока lease протухнет и reaper вернёт job в queued. + # lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом. + deadline = time.monotonic() + 120 + requeued = False + while time.monotonic() < deadline: + st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job) + if st == 'queued': + requeued = True + break + time.sleep(2) + check(requeued, 'reaper did not requeue stuck job (status=%s)' % st) + ok('reaper requeued the stuck job') + + # 6) Возобновляем worker: он должен доделать job до ready. + docker('kill', '-s', 'SIGCONT', 'worker') + att = wait_status(c, a, att['id'], 'ready', seconds=120) + ok('ready after unstuck') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/limits.py b/student_tests/limits.py new file mode 100644 index 00000000..9a0d6180 --- /dev/null +++ b/student_tests/limits.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""413, 415, 400 invalid_image — лимиты bytes/pixels/MIME.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import png_bytes, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + url = '/api/v1/chats/' + ch['id'] + '/attachments?filename=x.bin' + + # 413: > 10 MiB + big = b'\x89PNG\r\n\x1a\n' + b'\x00' * (11 * 1024 * 1024) + r = c.call('POST', url, 413, actor=a, raw=big, headers={'Content-Type': 'image/png'}) + ok('413 payload_too_large') + + # 415: text/plain + r = c.call('POST', url, 415, actor=a, raw=b'hello', headers={'Content-Type': 'text/plain'}) + ok('415 unsupported_media_type') + + # 415: клиент говорит image/png, но содержимое не PNG/JPEG по DetectContentType + r = c.call('POST', url, 415, actor=a, raw=b'\x00' * 100, headers={'Content-Type': 'image/png'}) + ok('415 mismatched content (client MIME not trusted)') + + # 400: PNG-сигнатура валидна (DetectContentType -> image/png), + # но IHDR/IDAT битые -> DecodeConfig падает -> invalid_image (400) + broken_png = b'\x89PNG\r\n\x1a\n' + b'\x00' * 40 + r = c.call('POST', url, 400, actor=a, raw=broken_png, headers={'Content-Type': 'image/png'}) + ok('400 invalid image (signature ok, header broken)') + + # 400: пиксели > MAX_IMAGE_PIXELS (если выставите 1e8 — этот тест не сработает; + # по умолчанию 40 MP, значит нужно 7000x7000 ≈ 49 MP) + huge = png_bytes(7000, 7000) + r = c.call('POST', url, 400, actor=a, raw=huge, headers={'Content-Type': 'image/png'}) + ok('400 too many pixels') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/outbox.py b/student_tests/outbox.py new file mode 100644 index 00000000..da941c73 --- /dev/null +++ b/student_tests/outbox.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Outbox: пауза outbox → 202 → unpause → ready; published_at=NULL в промежутке.""" +import sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + docker('pause', 'outbox') + try: + att = upload(c, a, ch, content) + check(att['status'] == 'queued', 'expected queued') + time.sleep(3) + published = psql("SELECT coalesce(published_at::text,'NULL') FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + check(published == 'NULL', 'outbox paused but published_at=%s' % published) + ok('outbox paused: published_at IS NULL, upload accepted') + finally: + docker('unpause', 'outbox') + + att = wait_status(c, a, att['id'], 'ready', seconds=90) + ok('after unpause: ready') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/redelivery.py b/student_tests/redelivery.py new file mode 100644 index 00000000..416a7c5e --- /dev/null +++ b/student_tests/redelivery.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Повторный XADD того же job_id не создаёт дублей вариантов.""" +import json, subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'ready') + before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']) + check(before == '3', 'expected 3 variants, got %s' % before) + + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + payload = json.dumps({ + "schema_version": 1, "job_id": job, "type": "attachment.process.v1", + "attachment_id": att['id'], "pipeline_version": 1, "attempt": 1, + "created_at": "2026-01-01T00:00:00Z", + }) + # Повторная доставка через Redis CLI. + docker('exec', '-T', 'redis', 'redis-cli', 'XADD', 'attachment.jobs', '*', 'payload', payload) + time.sleep(5) + after = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']) + check(after == '3', 'redelivery created duplicate variants: %s' % after) + ok('redelivery is no-op (still 3 non-original variants; original via object_key)') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/replay.py b/student_tests/replay.py new file mode 100644 index 00000000..e5c20ce7 --- /dev/null +++ b/student_tests/replay.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Replay failed → снова queued.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + # Форсим failed (битый файл проходит DecodeConfig, но не Decode). + content = Path('tests/fixtures/sample.png').read_bytes()[:-20] + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'failed', seconds=180) + ok('failed reached') + + c.call('POST', '/api/v1/attachments/' + att['id'] + '/replay', 202, actor=a) + st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + check(st == 'queued', 'after replay expected queued, got %s' % st) + ok('replay -> queued') + + # Replay дал одну новую попытку; ошибка permanent, поэтому снова failed при attempt=1. + att = wait_status(c, a, att['id'], 'failed', seconds=180) + atts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])) + check(atts == 1, 'after replay: permanent error must fail at attempt=1, got %d' % atts) + ok('replay reset attempt to 1; permanent error failed again — not retried') + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/retry.py b/student_tests/retry.py new file mode 100644 index 00000000..66ce29e4 --- /dev/null +++ b/student_tests/retry.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready.""" +import subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + # 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем. + root_user = open('secrets/minio_root_user.txt').read().strip() + root_pass = open('secrets/minio_root_password.txt').read().strip() + docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc', + 'http://localhost:9000', root_user, root_pass, '--api', 'S3v4') + + # 2) Пауза worker’а: job не будет взят до unpause. + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id']) + check(staging.startswith('staging/'), 'unexpected staging_key') + + # 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект. + # Если worker ещё не стартовал — GetObject вернёт NoSuchKey. + subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm', + 'loc/messenger-attachments/' + staging], + capture_output=True, text=True, check=True) + + # 4) Отдаём job worker’у. + docker('unpause', 'worker') + + # 5) Ждём, пока attempt >= 2. + job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 90 + attempt = 1 + while time.monotonic() < deadline: + attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job)) + if attempt >= 2: + break + time.sleep(1) + check(attempt >= 2, + 'retry did not happen on missing staging object (attempt=%d)' % attempt) + ok('transient error retried: attempt=%d' % attempt) + + # 6) Логи retry именно для нашего job_id. + logs = docker('logs', '--tail=500', 'worker') + lines = [l for l in logs.splitlines() + if ('retry job=' + job) in l] + check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:])) + ok('retry log lines:') + for l in lines: + print(' ' + l) + + # 7) Возвращаем staging — job должен пройти retry и стать ready. + # Перезаписываем объект с правильным содержимым. + # Через mc pipe: передаём байты на stdin. + r = subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', + 'mc', 'pipe', 'loc/messenger-attachments/' + staging], + input=content, capture_output=True) + check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode()) + + att = wait_status(c, a, att['id'], 'ready', seconds=180) + ok('recovered to ready after staging restored (attempt=%d)' % attempt) + + finally: + # На всякий случай — если упали в середине, worker не должен остаться на паузе. + subprocess.run(['docker', 'compose', 'unpause', 'worker'], + capture_output=True) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/staging-cleanup.py b/student_tests/staging-cleanup.py new file mode 100644 index 00000000..9edba4a6 --- /dev/null +++ b/student_tests/staging-cleanup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""После ready staging пуст (объект перенесён в final).""" +import sys +import subprocess +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, psql, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id']) + check(staging.startswith('staging/'), 'staging_key does not start with staging/') + att = wait_status(c, a, att['id'], 'ready') + root_user = open('secrets/minio_root_user.txt').read().strip() + root_pass = open('secrets/minio_root_password.txt').read().strip() + docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'local_check', + 'http://localhost:9000', root_user, root_pass, '--api', 'S3v4') + + r = subprocess.run( + ['docker', 'compose', 'exec', '-T', 's3', 'mc', 'stat', + 'local_check/messenger-attachments/' + staging], + capture_output=True, text=True) + combined = ((r.stdout or '') + (r.stderr or '')).lower() + check('does not exist' in combined or 'not found' in combined, + 'staging object still present: %s\n%s' % (staging, combined)) + ok('staging object removed after ready (mc rc=%d)' % r.returncode) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/states.py b/student_tests/states.py new file mode 100644 index 00000000..ff9ae2c5 --- /dev/null +++ b/student_tests/states.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""queued → processing → ready через остановку worker’а.""" +import time, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, docker, ok, fail + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + + docker('pause', 'worker') + try: + att = upload(c, a, ch, content) + check(att['status'] == 'queued', 'expected queued after upload') + ok('upload -> queued') + # Не должно перейти в processing, пока worker на паузе. + time.sleep(3) + att = c.call('GET', '/api/v1/attachments/' + att['id'], actor=a, schema='Attachment')['json'] + check(att['status'] == 'queued', 'worker paused but status changed to %s' % att['status']) + ok('status stays queued while worker paused') + finally: + docker('unpause', 'worker') + + att = wait_status(c, a, att['id'], 'ready', seconds=90) + names = {v['name'] for v in att['variants']} + # original в variants[] нет по контракту OpenAPI — он в object_key/оригинальном файле. + check(names == {'thumbnail', 'crop', 'optimized'}, + 'variants mismatch: %s' % names) + ok('ready with thumbnail/crop/optimized (original available via ?variant=original)') + + # original доступен отдельно — проверим, что эндпоинт отвечает 200. + r = c.call('GET', + '/api/v1/attachments/' + att['id'] + '/content?variant=original', + 200, actor=a) + check(len(r['bytes']) > 0, 'original empty') + ok('original served via ?variant=original (%d bytes)' % len(r['bytes'])) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file diff --git a/student_tests/variants.py b/student_tests/variants.py new file mode 100644 index 00000000..13aa2310 --- /dev/null +++ b/student_tests/variants.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""original неизменен, thumbnail 256x256, crop 128x128, optimized существует.""" +import hashlib, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests.client import Client, check +from student_tests.common import upload, wait_status, ok, fail +from student_tests.jpeg_probe import jpeg_size as dims_jpeg + +def main(): + c = Client() + a, b, e, ch = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att = upload(c, a, ch, content) + att = wait_status(c, a, att['id'], 'ready') + src_sha = hashlib.sha256(content).hexdigest() + + def get(v): + return c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (att['id'], v), + 200, actor=a)['bytes'] + + orig = get('original') + check(hashlib.sha256(orig).hexdigest() == src_sha, 'original changed') + ok('original byte-identical') + + th = get('thumbnail') + w, h = dims_jpeg(th) + check(max(w, h) == 256 and w <= 256 and h <= 256, + 'thumbnail is %dx%d, expected fit in 256x256' % (w, h)) + check(w * 160 == h * 256 if h == 160 else True, ...) # необязательно + # Проще всего — сверить с metadata: + thumb_meta = next(v for v in att['variants'] if v['name'] == 'thumbnail') + check((w, h) == (thumb_meta['width'], thumb_meta['height']), + 'thumbnail metadata %sx%s != actual %dx%d' % (thumb_meta['width'], thumb_meta['height'], w, h)) + ok('thumbnail %dx%d (metadata matches)' % (w, h)) + + cr = get('crop') + w, h = dims_jpeg(cr) + check((w, h) == (128, 128), 'crop is %dx%d' % (w, h)) + ok('crop 128x128 JPEG') + + opt = get('optimized') + check(len(opt) > 0, 'optimized empty') + # optimized может быть JPEG или WebP; проверяем, что не пусто и похоже на изображение + check(opt[:2] == b'\xff\xd8' or opt[:4] == b'RIFF', 'optimized is not JPEG/WebP') + ok('optimized present (%d bytes)' % len(opt)) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + fail(str(exc)) \ No newline at end of file