feat:report

This commit is contained in:
Оля Мороз
2026-11-16 20:30:12 +03:00
parent 90f407b174
commit c91ec7790f
37 changed files with 1159 additions and 209 deletions
+1 -4
View File
@@ -1,5 +1,5 @@
# --- build stage ---
FROM golang:1.22-alpine AS build
FROM golang:1.26-alpine AS build
WORKDIR /src
RUN apk add --no-cache ca-certificates
@@ -29,7 +29,4 @@ RUN chmod +x /app/migrate.sh
USER app
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \
CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1
ENTRYPOINT ["/app/messenger"]
+22 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover my_tests my_tests_quick my_tests_slow
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,24 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# Быстрые (<1 мин): статусы, лимиты, варианты, payload, DLQ, staging.
my_tests_quick:
python3 student_tests/states.py
python3 student_tests/limits.py
python3 student_tests/variants.py
python3 student_tests/broker-payload.py
python3 student_tests/staging-cleanup.py
python3 student_tests/dlq.py
python3 student_tests/redelivery.py
# Долгие (>2 мин): retry/backoff/failed/replay/outbox/lease/crash.
my_tests_slow:
python3 student_tests/outbox.py
python3 student_tests/failed.py
python3 student_tests/retry.py
python3 student_tests/replay.py
python3 student_tests/lease.py
python3 student_tests/crash-before-ack.py
my_tests: my_tests_quick my_tests_slow
+166 -19
View File
@@ -1,37 +1,184 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Морозова Ольга Андреевна, БВТ2303
- Целевая оценка: **4** (треки на 5 не заявлены)
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники: [README.md](README.md), [COURSE.md](COURSE.md), [contracts/openapi.json](contracts/openapi.json), документация Go, MinIO (mc), PostgreSQL, Redis Streams, Nginx, Docker.
- Библиотеки: Go 1.26 (stdlib), github.com/google/uuid v1.6.0, github.com/jackc/pgx/v5 v5.5.5, github.com/redis/go-redis/v9 v9.5.1, golang.org/x/crypto v0.23.0, github.com/minio/minio-go/v7 v7.0.72, golang.org/x/image v0.48.0. Тесты — Python 3 (stdlib).
- Помощь ИИ: Deepseek — написание и проверка кода.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Команды выполнялись в WSL2 (Ubuntu) на Windows 11. Требования: Docker 24+ (Compose v2), Python 3.9+ (для тестов), `make`.
Версии (фиксированные теги образов):
- `golang:1.26-alpine` — build stage
- `alpine:3.20` — runtime stage
- `postgres:16.3-alpine` — база данных
- `redis:7.2.5-alpine` — кэш сессий и брокер (Redis Streams)
- `nginx:1.27.0-alpine` — балансировщик
- `pgsty/silo:RELEASE.2026-10-26T00-00-00Z` — S3-совместимое хранилище (MinIO-compatible)
### Подготовка
```sh
git clone https://gitea.privcloud.tech/MiniLynx/mtusi2026lab5-fork
cd mtusi2026lab5-fork
# 1. Секреты (не входят в Git)
mkdir -p secrets
printf 'local-dev-postgres-password\n' > secrets/postgres_password.txt
printf 'postgres://messenger:local-dev-postgres-password@postgres:5432/messenger?sslmode=disable\n' > secrets/database_url.txt
printf 'redis://redis:6379/0\n' > secrets/cache_url.txt
printf 'minioadmin\n' > secrets/minio_root_user.txt
printf 'minioadmin-dev-password\n' > secrets/minio_root_password.txt
printf 'messenger-api\n' > secrets/s3_access_key_id.txt
printf 'messenger-api-dev-secret\n' > secrets/s3_secret_access_key.txt
chmod 600 secrets/*.txt
# 2. Локальный .env
cp .env.example .env
# ВАЖНО: В .env нужно исправить S3_ENDPOINT на s3:9000 (без http://)
sed -i 's|S3_ENDPOINT=.*|S3_ENDPOINT=s3:9000|' .env
# 3. Python для проверок
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements-dev.txt
# 4. Скрипты
chmod +x scripts/migrate.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh
```
### Сборка и запуск
```sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождаться /health/ready
docker compose ps
```
Для запуска с другим количеством реплик добавить `API_REPLICAS=$n` перед `docker compose --env-file .env up -d --build`, где `$n` - желаемое количество реплик.
### Тесты
```sh
make check
make validate PYTHON=.venv/bin/python
make test
# Persistence
make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Failover
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
./scripts/restore-api.sh
# Собственные тесты (pipeline, сбои, лимиты)
make my_tests
```
## Остановка (с сохранением volumes)
```sh
docker compose --env-file .env down --remove-orphans
```
### Намеренное удаление учебных данных
```sh
docker compose --env-file .env down --remove-orphans -v
rm -f secrets/*.txt
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
HTTP-клиент → Nginx (8080) → API (Go, x2, stateless)
│
├──[S3 PUT]→ staging/ (S3, приватный bucket)
├──[INSERT]→ Postgres: attachments (queued) + attachment_jobs (outbox)
└──[202 Accepted]
Outbox (x1) ⇄ Postgres (attachment_jobs.published_at IS NULL)
│
└──[XADD]→ Redis Streams (attachment.jobs)
Worker (x2) ⇄ Redis Streams (consumer group)
├──[SELECT ... FOR UPDATE SKIP LOCKED]→ Postgres (lease 60s)
├──[GET]→ staging/ → resize (thumb 256, crop 128, opt) →[PUT]→ final/
├──[INSERT variants + UPDATE ready]→ Postgres (транзакция)
└──[XACK]→ Redis
```
- **Асинхронный приём (уровень 3):** `POST /api/v1/chats/{id}/attachments` возвращает **202 Accepted** сразу после того, как файл сохранён в staging, а в БД одной транзакцией записаны метаданные `attachments` (status=`queued`) и `attachment_jobs` (outbox). Это гарантирует, что после ответа 202 работа не потеряется при рестарте API.
- **Outbox (уровень 4):** отдельный процесс `cmd/outbox` читает `attachment_jobs WHERE published_at IS NULL` и публикует в брокер. `published_at` ставится только после `XADD`. Падение между COMMIT и publish приводит к повторной публикации при следующем цикле; worker идемпотентен.
- **Брокер:** Redis Streams (тот же Redis, AOF `--appendonly yes`). Consumer Group `workers`, `XACK` после durable-эффекта. В сообщении только `job_id`, `attachment_id`, `pipeline_version`, `attempt` (никаких URL или filename).
- **Worker (уровень 3-4):** отдельный процесс, реплики 2. Забирает job через `ClaimJobByID` (lease 60s). Пишет `original` (server-side copy staging→final), `thumbnail` (256×256, fit), `crop` (128×128, center-cover) и `optimized` (JPEG q=70). Всё в одной транзакции с `attachment_variants` и `status='ready'`. `XACK` только после COMMIT.
- **Идемпотентность:** ключ эффекта — `(attachment_id, pipeline_version, variant)`. Повторная доставка того же job_id: `MarkReady` — no-op (если `status='done'`), `attachment_variants` — `ON CONFLICT DO NOTHING`. Также `VariantsExist` перед Claim.
- **Retry:** `RetryJob` различает permanent (`invalid_image` → сразу failed) и transient (`storage_error` → backoff `base * 2^(n-1) + jitter`, max 5 попыток). Финальный `failed` с безопасным `error_code`.
- **Зависшие job:** `RequeueStuck` в outbox и worker, каждые 30с: `status='processing' AND lease_expires_at < now()` → `queued`, `published_at=NULL`.
- **Replay:** `POST /api/v1/attachments/{id}/replay` сбрасывает `attempt=1`, `status='queued'`, `published_at=NULL` для failed-задач.
- **Очистка:** `MoveToFinal` удаляет staging после CopyObject. `ReconcileOrphans` (в API) чистит `attachment_gc` и «зависшие» staging-объекты без job.
### Что перенесено из lab4
- PostgreSQL, Argon2id, sessions (с кэшем в Redis), горизонтальное масштабирование, Nginx.
- Механизм идемпотентности `idempotency_keys` для сообщений.
- S3-хранилище и приватный bucket, `S3Store` с таймаутами и ретраями.
### Что изменено
- Миграция `0002_attachments.sql`: добавлены `attachment_variants`, `attachment_jobs` (outbox+lease), поля `staging_key`, `pipeline_version`, `ready_at`, `failed_at`. Таблица `attachments` теперь содержит `status IN ('queued','processing','ready','failed')`.
- `UploadAttachment` возвращает 202, статус `queued`; `DownloadAttachment` → 409 для не-ready.
- Добавлены `cmd/outbox`, `cmd/worker` (обработка изображений, `process.go`).
- `src/domain/broker.go` — обёртка Redis Streams.
- `compose.yaml`: сервисы `outbox` (1 реплика) и `worker` (2 реплики), миграции вынесены в отдельный сервис `migrate`.
- В `attachments.go`: `EnqueueAttachment`, `ClaimJob`, `ClaimJobByID`, `MarkReady`, `RetryJob`, `RequeueStuck`, `ReplayFailed`, `MarkFailedPermanent`, `GetVariantKey`, `SelectUnpublishedJobs`, `MarkPublished`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------ |
| Offline-структура и локальные refs | `make check` | PASS | [png](evidence/2_Tests_Passing.png) |
| Полная OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | [png](evidence/2_Tests_Passing.png) |
| Публичный smoke (lab5) | `make test` | PASS: 202, ready, thumbnail | [png](evidence/2_Tests_Passing.png) |
| Persistence: перезапуск API+БД+Redis+S3 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | [png](evidence/2_Tests_Passing.png) |
| Failover: остановка одной реплики API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | `observed_instances_before: 2`, `successful_writes: 75`, `first_recovery: 5.371s` | [png](evidence/2_Tests_Passing.png) |
| **Ур. 3:** Upload → 202 → queued (worker не нужен) | `student_tests/states.py` | queued → ready, 3 варианта + original через `?variant=original` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Broker и worker — отдельные процессы | `docker compose ps` | `outbox`, `worker-1`, `worker-2` | [png](evidence/1_Healthcheck.png) |
| **Ур. 3:** Статусы в API, лимиты и типы | `student_tests/limits.py` | 413, 415, 400 invalid_image | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** original неизменен, thumbnail по контракту | `student_tests/variants.py` | `byte-identical`, `thumbnail 256x160 (metadata matches)`, `crop 128x128` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Broker payload без filename/URL | `student_tests/broker-payload.py` | только id/version/attempt | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Очистка staging после ready | `student_tests/staging-cleanup.py` | `staging object removed after ready` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Реестр failed (DLQ) | `student_tests/dlq.py` | `invalid_image\|1` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 3:** Идемпотентность при повторной доставке | `student_tests/redelivery.py` | `still 3 non-original variants` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Outbox закрывает разрыв COMMIT→publish | `student_tests/outbox.py` | `published_at IS NULL` при паузе outbox → ready после unpause | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Permanent error без retry | `student_tests/failed.py` | `attempt=1, invalid_image — not retried` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Transient error → retry → ready | `student_tests/retry.py` | `attempt=2`, `retry log lines: ... backoff=2s jitter=451ms` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Replay failed | `student_tests/replay.py` | `replay -> queued`, повторный permanent fail | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Lease / reaper зависших | `student_tests/lease.py` | `reaper requeued the stuck job` → `ready` | [txt](evidence/my_tests_passing.txt) |
| **Ур. 4:** Crash после side effect до ack | `student_tests/crash-before-ack.py` | `recovery produced exactly 3 variants, no duplicates` | [txt](evidence/my_tests_passing.txt) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------- |
| Остановка одной реплики API | 0 потерянных подтверждённых записей, LB переключается | Failover PASS, 75 успешных сообщений | 5.371 с, потерь нет |
| Рестарт API/БД/Redis/S3 (volumes) | Вложение и сессия живы | Persistence PASS | ~30 с (запуск контейнеров), потерь нет |
| Отказ S3 | API возвращает 503, worker ретраит | `retry.py` PASS, `attempt=2`, backoff сработал | После восстановления S3 — ready |
| Пауза outbox (разрыв COMMIT→publish) | `published_at=NULL`, upload 202 | `outbox.py` PASS, job дошёл до ready после unpause | Задержка publish = период паузы, потерь нет |
| Пауза worker (зависший processing) | Lease истекает, reaper возвращает в queued | `lease.py` PASS, job вернулся в queued через ~60с + reaper | ~90 с, потерь нет |
| `SIGKILL` worker во время processing | Повторная доставка, без дублей вариантов | `crash-before-ack.py` PASS, 3 варианта, без дублей | ~100 с (lease + рестарт), потерь нет |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **At-least-once:** доставка Redis Streams + идемпотентный эффект ≠ exactly-once. При падении между XADD и MarkPublished возможна повторная публикация; повторный worker это корректно обрабатывает, но сообщение в Stream продублируется.
- **Replay** доступен через endpoint `POST /api/v1/attachments/{id}/replay` (расширение сверх OpenAPI). В отчёте зафиксировано, что это не меняет семантику обязательных путей.
- **Optimized** — JPEG q=70 вместо WebP (WebP-кодирование требует cgo-зависимостей, которых в alpine-сборке без CGO нет). Контракт допускает JPEG как «перекодированный» вариант; в отчёте явно указано.
- **Broker = Redis Streams** вместо RabbitMQ — обосновано тем, что Redis уже нужен для кэша сессий, AOF включён, это уменьшает количество контейнеров. Если Redis упадёт без AOF-реплики, неопубликованные задачи будут ждать в `attachment_jobs` (outbox) — они не потеряются.
- **DLQ** — пока реестр `attachment_jobs.status='failed'` + `last_error_code`, без отдельного UI. Для оператора — `SELECT` из БД и endpoint replay.
- **Не реализовано:** версионирование нескольких pipeline (трек 5), auto-scaling worker’а по queue lag (трек 1), fault-injection с версионированием (трек 2). Уровень 5 не заявлен.
- **Нагрузка** проверена только smoke + `make test`; полноценных замеров throughput/latency pipeline не делалось (не требуется для уровня 4).
+2 -1
View File
@@ -168,7 +168,8 @@ func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store,
return
}
} else {
sum, err := s3.PutBytes(ctx, finalOriginalKey, v.ContentType, v.Data)
key := "final/" + j.AttachmentID + "/" + v.Name
sum, err := s3.PutBytes(ctx, key, v.ContentType, v.Data)
if err != nil {
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
_ = broker.Ack(ctx, m.ID)
+33 -19
View File
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"image"
"image/jpeg"
"math"
"golang.org/x/image/draw"
)
@@ -29,7 +30,7 @@ func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) {
// processImage возвращает 4 варианта. Оригинал — byte-identical с входом.
// thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop),
// optimized — WebP q=80 без метаданных.
// optimized — JPEG q=70 без метаданных (WebP оставлен на усмотрение; см. отчёт).
func processImage(raw []byte, contentType string) ([]VariantSpec, error) {
src, _, err := image.Decode(bytes.NewReader(raw))
if err != nil {
@@ -46,14 +47,14 @@ func processImage(raw []byte, contentType string) ([]VariantSpec, error) {
Data: raw, SHA256: hex.EncodeToString(sum[:]),
})
// thumbnail 256x256 (fit).
// thumbnail: fit в 256x160 с сохранением пропорций (для 16:10 — 256x160).
th := resizeFit(src, 256, 256)
if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil {
return nil, err
}
// crop 128x128 (center crop).
cr := centerCrop(src, 128, 128)
// crop 128x128: центр-cover до меньшего квадрата.
cr := squareCover(src, 128)
if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil {
return nil, err
}
@@ -96,28 +97,41 @@ func resizeFit(src image.Image, maxW, maxH int) image.Image {
if scale > 1 {
scale = 1
}
nw, nh := int(float64(w)*scale), int(float64(h)*scale)
nw := int(math.Round(float64(w) * scale))
nh := int(math.Round(float64(h) * scale))
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil)
return dst
}
func centerCrop(src image.Image, w, h int) image.Image {
// squareCover масштабирует src так, чтобы покрыть квадрат size×size
// (min-масштаб по двум осям), затем центр-кроп до size×size.
// Результат — всегда ровно size×size.
func squareCover(src image.Image, size int) image.Image {
b := src.Bounds()
sw, sh := b.Dx(), b.Dy()
cw, ch := w, h
if sw < cw {
cw = sw
w, h := b.Dx(), b.Dy()
// scale — больший из двух, чтобы покрыть квадрат
scale := float64(size) / float64(w)
if float64(size)/float64(h) > scale {
scale = float64(size) / float64(h)
}
if sh < ch {
ch = sh
nw := int(math.Round(float64(w) * scale))
nh := int(math.Round(float64(h) * scale))
if nw < size {
nw = size
}
x0 := (sw - cw) / 2
y0 := (sh - ch) / 2
dst := image.NewRGBA(image.Rect(0, 0, w, h))
// масштабируем crop-область в целевой размер (иначе мелкие картинки не влезут).
draw.CatmullRom.Scale(dst, dst.Bounds(),
src, image.Rect(b.Min.X+x0, b.Min.Y+y0, b.Min.X+x0+cw, b.Min.Y+y0+ch),
draw.Over, nil)
if nh < size {
nh = size
}
scaled := image.NewRGBA(image.Rect(0, 0, nw, nh))
draw.CatmullRom.Scale(scaled, scaled.Bounds(), src, b, draw.Over, nil)
// center-crop до size×size
x0 := (nw - size) / 2
y0 := (nh - size) / 2
dst := image.NewRGBA(image.Rect(0, 0, size, size))
draw.Draw(dst, dst.Bounds(), scaled,
image.Pt(x0, y0), draw.Src)
return dst
}
+9
View File
@@ -10,6 +10,7 @@ x-api: &api
redis: { condition: service_healthy }
s3: { condition: service_healthy }
s3-init: { condition: service_completed_successfully }
migrate: { condition: service_completed_successfully }
networks: [backend]
services:
@@ -91,6 +92,12 @@ services:
limits:
cpus: "${API_CPU_PER_REPLICA:-0.5}"
memory: "${API_MEM_PER_REPLICA:-128M}"
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
interval: 10s
timeout: 3s
start_period: 20s
retries: 5
outbox:
<<: *api
@@ -98,6 +105,7 @@ services:
depends_on:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
deploy:
replicas: 1
resources:
@@ -110,6 +118,7 @@ services:
postgres: { condition: service_healthy }
redis: { condition: service_healthy }
s3: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
deploy:
replicas: 2
resources:
Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 94 KiB

+56
View File
@@ -0,0 +1,56 @@
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$ make my_tests
python3 student_tests/states.py
upload -> queued
status stays queued while worker paused
ready with thumbnail/crop/optimized (original available via ?variant=original)
original served via ?variant=original (111817 bytes)
python3 student_tests/limits.py
413 payload_too_large
415 unsupported_media_type
415 mismatched content (client MIME not trusted)
400 invalid image (signature ok, header broken)
400 too many pixels
python3 student_tests/variants.py
original byte-identical
thumbnail 256x160 (metadata matches)
crop 128x128 JPEG
optimized present (4818 bytes)
python3 student_tests/broker-payload.py
stream sample:
1794848167433-0
payload
{"schema_version":1,"job_id":"ee59551f-c85f-411a-b4d9-6076f31fde8f","type":"attachment.process.v1","attachment_id":"1175cf24-2984-4800-b84d-1aa702ecd7bd","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:07Z"}
1794848176445-0
payload
{"schema_version":1,"job_id":"b7792daf-6123-4c7a-9566-b2fc4a53a614","type":"attachment.process.v1","attachment_id":"05ef10e7-19c4-47ef-9c68-a2e09eb8338e","pipeline_version":1,"attempt":1,"created_at":"2026-11-16T16:56:16Z"}
broker payload has only ids/version/attempt
python3 student_tests/staging-cleanup.py
staging object removed after ready (mc rc=1)
python3 student_tests/dlq.py
failed registry:
b1ccbeca-ae22-44f1-837c-868d2165889c|invalid_image|1
python3 student_tests/redelivery.py
redelivery is no-op (still 3 non-original variants; original via object_key)
python3 student_tests/outbox.py
outbox paused: published_at IS NULL, upload accepted
after unpause: ready
python3 student_tests/failed.py
failed immediately (attempt=1), error_code=invalid_image — permanent, not retried
python3 student_tests/retry.py
transient error retried: attempt=2
retry log lines:
worker-1 | 2026/11/16 16:56:48 retry job=64072f3d-6b9e-4101-b31b-7e9a2f51bac3 attempt=2 next_attempt=2026-11-16T16:56:50.964432673Z backoff=2s jitter=451.35473ms code=storage_error
recovered to ready after staging restored (attempt=2)
python3 student_tests/replay.py
failed reached
replay -> queued
replay reset attempt to 1; permanent error failed again — not retried
python3 student_tests/lease.py
worker stopped at processing (job=da0adcf7-64a7-427c-a50b-438e9d688bb7, lease_expires_at=2026-11-16 16:57:59.829609+00)
reaper requeued the stuck job
ready after unstuck
python3 student_tests/crash-before-ack.py
worker killed during processing
recovery produced exactly 3 non-original variants, no duplicates (original in object_key)
(.venv) linux@DESKTOP-7TS8I4A:/mnt/c/MiniLynx/Forth/VP/lab5/mtusi2026lab5-fork$
+58
View File
@@ -0,0 +1,58 @@
-- ============================================================
-- Core: users, sessions, chats, members, messages, idempotency
-- ============================================================
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
status_code INTEGER NOT NULL,
response JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_keys (created_at);
-31
View File
@@ -1,31 +0,0 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Индекс истории: ровно под ORDER BY (created_at, id) и keyset-курсор.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
+88
View File
@@ -0,0 +1,88 @@
-- ============================================================
-- Attachments: metadata, variants, jobs/outbox, GC
-- ============================================================
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
object_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0 AND size_bytes <= 10485760),
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
width INTEGER NOT NULL CHECK (width >= 0),
height INTEGER NOT NULL CHECK (height >= 0),
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','processing','ready','failed','pending')),
error_code TEXT,
-- поля из 0006
pipeline_version INTEGER NOT NULL DEFAULT 1,
staging_key TEXT,
ready_at TIMESTAMPTZ,
failed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id);
-- Для reconciler/поиска «зависших» pending-записей:
CREATE INDEX IF NOT EXISTS idx_attachments_pending
ON attachments (created_at) WHERE status <> 'ready';
-- Связь сообщение ↔ вложения. FK на элементы массива Postgres не умеет,
-- целостность — на уровне приложения (или триггера, если нужно).
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
-- Журнал компенсации: объекты S3, которые могли остаться без метаданных.
CREATE TABLE IF NOT EXISTS attachment_gc (
object_key TEXT PRIMARY KEY,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Idempotency эффекта: (attachment_id, pipeline_version, variant)
-- content_type без CHECK: варианты могут быть webp и т.п.
-- sha256 — text+CHECK, чтобы не ловить padding от char(64).
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
variant TEXT NOT NULL,
object_key TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
-- Outbox + очередь job-ов
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id UUID PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','processing','done','failed')),
-- outbox
published_at TIMESTAMPTZ,
publish_attempts INTEGER NOT NULL DEFAULT 0,
-- lease
lease_owner TEXT,
lease_expires_at TIMESTAMPTZ,
-- retry
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию
);
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished
ON attachment_jobs (next_attempt_at)
WHERE published_at IS NULL AND status = 'queued';
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
ON attachment_jobs (lease_expires_at)
WHERE status = 'processing';
-9
View File
@@ -1,9 +0,0 @@
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions (expires_at);
@@ -1,5 +1,5 @@
-- Индекс B для «стоимости записи»:
-- покрывающий индекс, включающий text — растёт быстрее, помогает только
-- покрывающим запросам.
-- покрывающим запросам. Накатывать отдельно, чтобы сравнить с индексом A.
CREATE INDEX IF NOT EXISTS idx_messages_chat_covering
ON messages (chat_id, created_at, id) INCLUDE (sender_id, text);
-14
View File
@@ -1,14 +0,0 @@
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
status_code INTEGER NOT NULL,
response JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_keys (created_at);
-30
View File
@@ -1,30 +0,0 @@
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
object_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
width INTEGER NOT NULL CHECK (width > 0),
height INTEGER NOT NULL CHECK (height > 0),
status TEXT NOT NULL DEFAULT 'ready' CHECK (status IN ('pending','ready','failed')),
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id);
-- Для reconciler/поиска «зависших» pending-записей:
CREATE INDEX IF NOT EXISTS idx_attachments_pending ON attachments (created_at) WHERE status <> 'ready';
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
-- Журнал компенсации: объекты S3, которые могли остаться без метаданных.
CREATE TABLE IF NOT EXISTS attachment_gc (
object_key TEXT PRIMARY KEY,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-49
View File
@@ -1,49 +0,0 @@
-- staging → final prefix; job-очередь и lease
ALTER TABLE attachments
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN IF NOT EXISTS staging_key text,
ADD COLUMN IF NOT EXISTS ready_at timestamptz,
ADD COLUMN IF NOT EXISTS failed_at timestamptz,
ADD COLUMN IF NOT EXISTS error_code text;
-- idempotency эффекта: (attachment_id, pipeline_version, variant)
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
variant text NOT NULL,
object_key text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width int NOT NULL,
height int NOT NULL,
sha256 char(64) NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
-- outbox + очередь job-ов
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
attempt int NOT NULL DEFAULT 1,
status text NOT NULL DEFAULT 'queued', -- queued|processing|done|failed
-- outbox
published_at timestamptz,
publish_attempts int NOT NULL DEFAULT 0,
-- lease
lease_owner text,
lease_expires_at timestamptz,
-- retry
next_attempt_at timestamptz NOT NULL DEFAULT now(),
last_error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию
);
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished
ON attachment_jobs (next_attempt_at)
WHERE published_at IS NULL AND status = 'queued';
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
ON attachment_jobs (lease_expires_at)
WHERE status = 'processing';
+15 -3
View File
@@ -6,8 +6,20 @@ URL_FILE="${DATABASE_URL_FILE:-/run/secrets/database_url}"
DATABASE_URL="$(cat "$URL_FILE")"
PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1 -q"
for f in migrations/*.sql; do
echo "applying $f"
$PSQL -f "$f"
echo "waiting for postgres..."
for i in $(seq 1 60); do
if psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null 2>&1; then
echo "postgres is ready"
break
fi
echo " attempt $i: not ready yet"
sleep 1
done
psql "$DATABASE_URL" -c 'SELECT 1' >/dev/null
for f in /app/migrations/*.sql; do
echo "applying $f"
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$f"
done
echo "migrations done"
+8 -1
View File
@@ -459,7 +459,14 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
pendingID, err := h.Store.CreateAttachmentPending(
r.Context(), chatID, me.ID, stagingKey, filename)
if err != nil {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
switch {
case errors.Is(err, domain.ErrInvalid):
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid attachment metadata")
case errors.Is(err, domain.ErrTimeout):
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database timeout")
default:
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
}
return
}
+1 -1
View File
@@ -9,4 +9,4 @@ const (
const PipelineVersion = 1
var RequiredVariants = []string{"original", "thumbnail", "crop", "optimized"}
var RequiredVariants = []string{"thumbnail", "crop", "optimized"}
+40 -23
View File
@@ -3,6 +3,7 @@ package domain
import (
"context"
"errors"
"log"
"math/rand"
"time"
@@ -48,12 +49,12 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
var errCode *string
err := s.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
@@ -64,10 +65,11 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
a.Variants = []Variant{}
rows, err := s.pool.Query(ctx,
`SELECT variant, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2
ORDER BY variant`,
a.ID, a.PipelineVersion)
FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2
AND variant = ANY($3)
ORDER BY variant`,
a.ID, a.PipelineVersion, RequiredVariants)
if err != nil {
return nil, classifyDBError(err)
}
@@ -111,10 +113,13 @@ func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string,
return ErrInvalid
}
seen[id] = true
var ok bool
var status string
err := s.pool.QueryRow(ctx,
`SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`,
id, chatID).Scan(&ok)
`SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`,
id, chatID).Scan(&status)
if err == nil && status == AttachmentFailed {
return ErrInvalid
}
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
@@ -199,10 +204,10 @@ func (s *Store) EnqueueAttachment(
pipeline_version=$7, status='queued', updated_at=now()
WHERE id=$1 AND status='queued'
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at`,
status, error_code, created_at, updated_at, pipeline_version`,
attachmentID, contentType, size, sha, w, h, pipelineVersion,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes,
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt)
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &a.PipelineVersion)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
@@ -312,10 +317,13 @@ func (s *Store) MarkReady(
}
for _, v := range variants {
if v.Name == "original" {
continue // original живёт в attachments.object_key, не в variants
}
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_variants
(attachment_id, pipeline_version, variant, object_key,
content_type, size_bytes, width, height, sha256)
(attachment_id, pipeline_version, variant, object_key,
content_type, size_bytes, width, height, sha256)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`,
attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name,
@@ -374,6 +382,8 @@ func (s *Store) RetryJob(
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
log.Printf("retry job=%s exhausted attempts=%d code=%s",
jobID, attempt, errorCode)
return true, nil
}
@@ -388,6 +398,7 @@ func (s *Store) RetryJob(
status = 'queued',
last_error_code=$2,
next_attempt_at=$3,
published_at=NULL,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID, errorCode, next); err != nil {
return false, classifyDBError(err)
@@ -395,16 +406,21 @@ func (s *Store) RetryJob(
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
// Логируем зафиксированный retry — используется в student-tests/backoff.py
log.Printf("retry job=%s attempt=%d next_attempt=%s backoff=%s jitter=%s code=%s",
jobID, attempt+1, next.UTC().Format(time.RFC3339Nano),
backoff, jitter, errorCode)
return false, nil
}
// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox.
func (s *Store) RequeueStuck(ctx context.Context) (int64, error) {
tag, err := s.pool.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
next_attempt_at = now(), updated_at = now()
WHERE status='processing' AND lease_expires_at < now()`)
UPDATE attachment_jobs
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
next_attempt_at = now(), published_at = NULL, updated_at = now()
WHERE status='processing' AND lease_expires_at < now()`)
if err != nil {
return 0, classifyDBError(err)
}
@@ -421,9 +437,10 @@ func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error {
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', attempt=1, next_attempt_at=now(),
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
SET status='queued', attempt=1, next_attempt_at=now(),
published_at=NULL,
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
if err != nil {
return classifyDBError(err)
}
+14 -3
View File
@@ -526,15 +526,26 @@ func (s *Store) AddMessageIdempotent(
return nil, nil, false, ErrInvalid
}
seen[aid] = true
var ok bool
var (
status string
objectKey string
)
// Вложение из этого же чата; не обязательно ready — клиент увидит картинку,
// когда worker завершит обработку. failed — принимаем: сообщение уже создано,
// UI покажет «failed» через GET /attachments/{id}.
if err := tx.QueryRow(ctx,
`SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`,
aid, chatID).Scan(&ok); err != nil {
`SELECT status, object_key FROM attachments WHERE id=$1 AND chat_id=$2`,
aid, chatID).Scan(&status, &objectKey); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil, false, ErrNotFound
}
return nil, nil, false, classifyDBError(err)
}
if status == AttachmentFailed {
// Нельзя прикреплять заведомо неудачные вложения — иначе UI будет
// навсегда показывать сломанную картинку без пути к replay.
return nil, nil, false, ErrInvalid
}
}
// 2) Уже был такой ключ?
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""В Stream нет filename/URL/секретов — только id/version/attempt."""
import json, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
upload(c, a, ch, content)
raw = docker('exec', '-T', 'redis', 'redis-cli', '--raw', 'XRANGE', 'attachment.jobs', '-', '+')
ok('stream sample:\n' + raw)
lower = raw.lower()
for bad in ('filename', 'sample.png', 'staging/', 'final/', 'http', 'secret'):
check(bad not in lower, 'broker payload contains forbidden token: %s' % bad)
ok('broker payload has only ids/version/attempt')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+73
View File
@@ -0,0 +1,73 @@
"""Shared helpers for student lab5 tests; no tokens are logged."""
import base64, json, os, subprocess, sys, time, uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, Failure, check # noqa: E402
def fail(msg):
print('FAIL:', msg)
raise SystemExit(1)
def ok(msg='PASS'):
print(msg)
def docker(*args, check_rc=True):
"""Run docker compose <args>; return stdout. Raises if rc!=0 and check_rc."""
rc = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True)
if check_rc and rc.returncode != 0:
raise Failure('docker compose %s: %s' % (' '.join(args), rc.stderr.strip()))
return rc.stdout
def psql(sql):
return docker('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-tAc', sql).strip()
def upload(c, actor, chat, content, filename='sample.png', ct='image/png'):
return c.call('POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
202, actor=actor, raw=content, headers={'Content-Type': ct},
schema='Attachment')['json']
def wait_status(c, actor, att_id, want, seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
if att['status'] == want:
return att
if att['status'] == 'failed' and want != 'failed':
fail('attachment failed: ' + str(att.get('error_code')))
time.sleep(0.25)
fail('attachment did not reach %s in %ss' % (want, seconds))
def png_bytes(w, h, seed=0):
"""Generate a valid PNG without external libs.
Быстро и с малой памятью: одна строка пикселей переиспользуется.
Для 7000x7000 занимает ~1-2 секунды и ~150 MiB RAM.
"""
import struct, zlib
# одна строка: filter byte 0 + w * 3 байта RGB
row = bytearray(1 + w * 3)
for x in range(w):
row[1 + x*3] = (x * 7 + seed) & 0xFF
row[2 + x*3] = (x * 11) & 0xFF
row[3 + x*3] = (x * 13) & 0xFF
row = bytes(row)
# zlib потоково, без накопления всего raw
co = zlib.compressobj(level=6)
idat_parts = []
for y in range(h):
idat_parts.append(co.compress(row))
idat_parts.append(co.flush())
idat = b''.join(idat_parts)
def chunk(t, d):
return struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff)
ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n'
+ chunk(b'IHDR', ihdr)
+ chunk(b'IDAT', idat)
+ chunk(b'IEND', b''))
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""kill -9 worker после S3 write до ack → повтор не создаёт дублей."""
import sys, time, signal
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = png_bytes(4000, 4000)
# Сначала готовим staging+job, но без запуска worker’а.
docker('stop', 'worker')
att = upload(c, a, ch, content)
# Запускаем worker — он возьмёт processing и, скорее всего, быстро закончит.
docker('start', 'worker')
# Ловим момент processing и убиваем процесс.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
if st == 'processing':
break
time.sleep(0.1)
docker('kill', '-s', 'SIGKILL', 'worker')
ok('worker killed during processing')
# После lease (60s) + reaper (30s) job должна снова стать queued.
time.sleep(100)
docker('start', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=120)
n = int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id']))
check(n == 3, 'expected exactly 3 variants, got %d' % n)
ok('recovery produced exactly 3 non-original variants, no duplicates (original in object_key)')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Реестр failed доступен оператору (SELECT из attachment_jobs)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
att = upload(c, a, ch, content)
wait_status(c, a, att['id'], 'failed', seconds=180)
rows = psql(
"SELECT attachment_id, last_error_code, attempt FROM attachment_jobs "
"WHERE status='failed' ORDER BY updated_at DESC LIMIT 5")
check(rows, 'no failed jobs visible to operator')
ok('failed registry:\n' + rows)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Битый файл: 5 attempt → failed, error_code=invalid_image."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
# Заведомо валидный PNG в API (MIME проходит), но worker получит
# битый заголовок, если подменить staging — здесь проверяем
# реалистичный сценарий: JPEG с обрезанным хвостом.
content = (Path('tests/fixtures/sample.png')).read_bytes()[:-20]
att = upload(c, a, ch, content)
# API пропустит (DecodeConfig проходит), worker упадёт на Decode.
att = wait_status(c, a, att['id'], 'failed', seconds=180)
check(att.get('error_code') == 'invalid_image', 'unexpected error_code: %s' % att.get('error_code'))
attempts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
# invalid_image — постоянная ошибка, worker не должен ретраить.
check(attempts == 1, 'permanent error must not be retried: attempt=%d' % attempts)
ok('failed immediately (attempt=1), error_code=invalid_image — permanent, not retried')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+28
View File
@@ -0,0 +1,28 @@
"""Мини-парсер SOF для JPEG; без внешних библиотек."""
from tests.client import Failure, check
def jpeg_size(data):
check(data[:2] == b'\xff\xd8', 'thumbnail is not JPEG')
i = 2
sof = {0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf}
while i < len(data):
check(data[i] == 0xff, 'invalid JPEG marker')
while i < len(data) and data[i] == 0xff:
i += 1
check(i < len(data), 'truncated JPEG marker')
marker = data[i]
i += 1
if marker in {0xd9, 0xda}:
break
if marker in {0x01, 0xd8} or 0xd0 <= marker <= 0xd7:
continue
check(i + 2 <= len(data), 'truncated JPEG segment')
length = int.from_bytes(data[i:i + 2], 'big')
check(length >= 2 and i + length <= len(data), 'invalid JPEG segment length')
if marker in sof:
check(length >= 8, 'invalid JPEG SOF')
return (int.from_bytes(data[i + 5:i + 7], 'big'),
int.from_bytes(data[i + 3:i + 5], 'big'))
i += length
raise Failure('JPEG dimensions not found')
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Reaper восстанавливает job после паузы worker’а дольше lease (60с)."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = png_bytes(4000, 4000)
# 1) Готовим job, но worker не даём её взять.
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
finally:
pass # unpause ниже — после того, как настроим SIGSTOP
# 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP.
# Используем docker kill --signal=SIGSTOP, чтобы остановить процесс
# (не контейнер). Контейнер "жив", но worker заморожен.
docker('unpause', 'worker')
# 3) Ждём processing; как только взял — сразу SIGSTOP.
deadline = time.monotonic() + 15
job = None
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
if st == 'processing':
# Останавливаем процесс, пока он внутри processing.
docker('kill', '-s', 'SIGSTOP', 'worker')
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
break
if st == 'done':
fail('job finished too fast to simulate stuck worker')
time.sleep(0.02)
check(job is not None, 'worker never entered processing')
# 4) Фиксируем lease_expires_at на момент SIGSTOP.
lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job)
ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease))
# 5) Ждём, пока lease протухнет и reaper вернёт job в queued.
# lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом.
deadline = time.monotonic() + 120
requeued = False
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job)
if st == 'queued':
requeued = True
break
time.sleep(2)
check(requeued, 'reaper did not requeue stuck job (status=%s)' % st)
ok('reaper requeued the stuck job')
# 6) Возобновляем worker: он должен доделать job до ready.
docker('kill', '-s', 'SIGCONT', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=120)
ok('ready after unstuck')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""413, 415, 400 invalid_image — лимиты bytes/pixels/MIME."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import png_bytes, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
url = '/api/v1/chats/' + ch['id'] + '/attachments?filename=x.bin'
# 413: > 10 MiB
big = b'\x89PNG\r\n\x1a\n' + b'\x00' * (11 * 1024 * 1024)
r = c.call('POST', url, 413, actor=a, raw=big, headers={'Content-Type': 'image/png'})
ok('413 payload_too_large')
# 415: text/plain
r = c.call('POST', url, 415, actor=a, raw=b'hello', headers={'Content-Type': 'text/plain'})
ok('415 unsupported_media_type')
# 415: клиент говорит image/png, но содержимое не PNG/JPEG по DetectContentType
r = c.call('POST', url, 415, actor=a, raw=b'\x00' * 100, headers={'Content-Type': 'image/png'})
ok('415 mismatched content (client MIME not trusted)')
# 400: PNG-сигнатура валидна (DetectContentType -> image/png),
# но IHDR/IDAT битые -> DecodeConfig падает -> invalid_image (400)
broken_png = b'\x89PNG\r\n\x1a\n' + b'\x00' * 40
r = c.call('POST', url, 400, actor=a, raw=broken_png, headers={'Content-Type': 'image/png'})
ok('400 invalid image (signature ok, header broken)')
# 400: пиксели > MAX_IMAGE_PIXELS (если выставите 1e8 — этот тест не сработает;
# по умолчанию 40 MP, значит нужно 7000x7000 ≈ 49 MP)
huge = png_bytes(7000, 7000)
r = c.call('POST', url, 400, actor=a, raw=huge, headers={'Content-Type': 'image/png'})
ok('400 too many pixels')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Outbox: пауза outbox → 202 → unpause → ready; published_at=NULL в промежутке."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
docker('pause', 'outbox')
try:
att = upload(c, a, ch, content)
check(att['status'] == 'queued', 'expected queued')
time.sleep(3)
published = psql("SELECT coalesce(published_at::text,'NULL') FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
check(published == 'NULL', 'outbox paused but published_at=%s' % published)
ok('outbox paused: published_at IS NULL, upload accepted')
finally:
docker('unpause', 'outbox')
att = wait_status(c, a, att['id'], 'ready', seconds=90)
ok('after unpause: ready')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Повторный XADD того же job_id не создаёт дублей вариантов."""
import json, subprocess, sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'ready')
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
check(before == '3', 'expected 3 variants, got %s' % before)
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
payload = json.dumps({
"schema_version": 1, "job_id": job, "type": "attachment.process.v1",
"attachment_id": att['id'], "pipeline_version": 1, "attempt": 1,
"created_at": "2026-01-01T00:00:00Z",
})
# Повторная доставка через Redis CLI.
docker('exec', '-T', 'redis', 'redis-cli', 'XADD', 'attachment.jobs', '*', 'payload', payload)
time.sleep(5)
after = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att['id'])
check(after == '3', 'redelivery created duplicate variants: %s' % after)
ok('redelivery is no-op (still 3 non-original variants; original via object_key)')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Replay failed → снова queued."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
# Форсим failed (битый файл проходит DecodeConfig, но не Decode).
content = Path('tests/fixtures/sample.png').read_bytes()[:-20]
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'failed', seconds=180)
ok('failed reached')
c.call('POST', '/api/v1/attachments/' + att['id'] + '/replay', 202, actor=a)
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
check(st == 'queued', 'after replay expected queued, got %s' % st)
ok('replay -> queued')
# Replay дал одну новую попытку; ошибка permanent, поэтому снова failed при attempt=1.
att = wait_status(c, a, att['id'], 'failed', seconds=180)
atts = int(psql("SELECT attempt FROM attachment_jobs WHERE attachment_id='%s'" % att['id']))
check(atts == 1, 'after replay: permanent error must fail at attempt=1, got %d' % atts)
ok('replay reset attempt to 1; permanent error failed again — not retried')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Временная ошибка S3: attempt растёт, backoff+jitter, затем ready."""
import subprocess, sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
# 1) Настраиваем mc с кредами — иначе удалить/положить объект не сможем.
root_user = open('secrets/minio_root_user.txt').read().strip()
root_pass = open('secrets/minio_root_password.txt').read().strip()
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'loc',
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
# 2) Пауза worker’а: job не будет взят до unpause.
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
check(staging.startswith('staging/'), 'unexpected staging_key')
# 3) Удаляем staging: это гарантирует, что worker НЕ прочитает объект.
# Если worker ещё не стартовал — GetObject вернёт NoSuchKey.
subprocess.run(
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'rm',
'loc/messenger-attachments/' + staging],
capture_output=True, text=True, check=True)
# 4) Отдаём job worker’у.
docker('unpause', 'worker')
# 5) Ждём, пока attempt >= 2.
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
deadline = time.monotonic() + 90
attempt = 1
while time.monotonic() < deadline:
attempt = int(psql("SELECT attempt FROM attachment_jobs WHERE job_id='%s'" % job))
if attempt >= 2:
break
time.sleep(1)
check(attempt >= 2,
'retry did not happen on missing staging object (attempt=%d)' % attempt)
ok('transient error retried: attempt=%d' % attempt)
# 6) Логи retry именно для нашего job_id.
logs = docker('logs', '--tail=500', 'worker')
lines = [l for l in logs.splitlines()
if ('retry job=' + job) in l]
check(len(lines) >= 1, 'no retry log lines for job %s:\n%s' % (job, logs[-1500:]))
ok('retry log lines:')
for l in lines:
print(' ' + l)
# 7) Возвращаем staging — job должен пройти retry и стать ready.
# Перезаписываем объект с правильным содержимым.
# Через mc pipe: передаём байты на stdin.
r = subprocess.run(
['docker', 'compose', 'exec', '-T', 's3',
'mc', 'pipe', 'loc/messenger-attachments/' + staging],
input=content, capture_output=True)
check(r.returncode == 0, 'failed to restore staging: %s' % r.stderr.decode())
att = wait_status(c, a, att['id'], 'ready', seconds=180)
ok('recovered to ready after staging restored (attempt=%d)' % attempt)
finally:
# На всякий случай — если упали в середине, worker не должен остаться на паузе.
subprocess.run(['docker', 'compose', 'unpause', 'worker'],
capture_output=True)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""После ready staging пуст (объект перенесён в final)."""
import sys
import subprocess
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
staging = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
check(staging.startswith('staging/'), 'staging_key does not start with staging/')
att = wait_status(c, a, att['id'], 'ready')
root_user = open('secrets/minio_root_user.txt').read().strip()
root_pass = open('secrets/minio_root_password.txt').read().strip()
docker('exec', '-T', 's3', 'mc', 'alias', 'set', 'local_check',
'http://localhost:9000', root_user, root_pass, '--api', 'S3v4')
r = subprocess.run(
['docker', 'compose', 'exec', '-T', 's3', 'mc', 'stat',
'local_check/messenger-attachments/' + staging],
capture_output=True, text=True)
combined = ((r.stdout or '') + (r.stderr or '')).lower()
check('does not exist' in combined or 'not found' in combined,
'staging object still present: %s\n%s' % (staging, combined))
ok('staging object removed after ready (mc rc=%d)' % r.returncode)
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""queued → processing → ready через остановку worker’а."""
import time, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, docker, ok, fail
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
check(att['status'] == 'queued', 'expected queued after upload')
ok('upload -> queued')
# Не должно перейти в processing, пока worker на паузе.
time.sleep(3)
att = c.call('GET', '/api/v1/attachments/' + att['id'], actor=a, schema='Attachment')['json']
check(att['status'] == 'queued', 'worker paused but status changed to %s' % att['status'])
ok('status stays queued while worker paused')
finally:
docker('unpause', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=90)
names = {v['name'] for v in att['variants']}
# original в variants[] нет по контракту OpenAPI — он в object_key/оригинальном файле.
check(names == {'thumbnail', 'crop', 'optimized'},
'variants mismatch: %s' % names)
ok('ready with thumbnail/crop/optimized (original available via ?variant=original)')
# original доступен отдельно — проверим, что эндпоинт отвечает 200.
r = c.call('GET',
'/api/v1/attachments/' + att['id'] + '/content?variant=original',
200, actor=a)
check(len(r['bytes']) > 0, 'original empty')
ok('original served via ?variant=original (%d bytes)' % len(r['bytes']))
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""original неизменен, thumbnail 256x256, crop 128x128, optimized существует."""
import hashlib, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, ok, fail
from student_tests.jpeg_probe import jpeg_size as dims_jpeg
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = Path('tests/fixtures/sample.png').read_bytes()
att = upload(c, a, ch, content)
att = wait_status(c, a, att['id'], 'ready')
src_sha = hashlib.sha256(content).hexdigest()
def get(v):
return c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (att['id'], v),
200, actor=a)['bytes']
orig = get('original')
check(hashlib.sha256(orig).hexdigest() == src_sha, 'original changed')
ok('original byte-identical')
th = get('thumbnail')
w, h = dims_jpeg(th)
check(max(w, h) == 256 and w <= 256 and h <= 256,
'thumbnail is %dx%d, expected fit in 256x256' % (w, h))
check(w * 160 == h * 256 if h == 160 else True, ...) # необязательно
# Проще всего — сверить с metadata:
thumb_meta = next(v for v in att['variants'] if v['name'] == 'thumbnail')
check((w, h) == (thumb_meta['width'], thumb_meta['height']),
'thumbnail metadata %sx%s != actual %dx%d' % (thumb_meta['width'], thumb_meta['height'], w, h))
ok('thumbnail %dx%d (metadata matches)' % (w, h))
cr = get('crop')
w, h = dims_jpeg(cr)
check((w, h) == (128, 128), 'crop is %dx%d' % (w, h))
ok('crop 128x128 JPEG')
opt = get('optimized')
check(len(opt) > 0, 'optimized empty')
# optimized может быть JPEG или WebP; проверяем, что не пусто и похоже на изображение
check(opt[:2] == b'\xff\xd8' or opt[:4] == b'RIFF', 'optimized is not JPEG/WebP')
ok('optimized present (%d bytes)' % len(opt))
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))