diff --git a/.dockerignore b/.dockerignore index ef48c580..4fc15f64 100644 --- a/.dockerignore +++ b/.dockerignore @@ -31,3 +31,8 @@ lab.json .vscode .agents .codex + +**/.env +**/.env.* +**/.idea +**/.vscode diff --git a/Dockerfile b/Dockerfile index 7cd6ea0a..69970d6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder -RUN GOBIN=/out CGO_ENABLED=0 go install \ +RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install \ -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' \ github.com/pressly/goose/v3/cmd/goose@v3.27.1 @@ -25,6 +25,7 @@ COPY contracts ./contracts COPY dto ./dto COPY internal ./internal COPY migrations ./migrations +ENV GOFLAGS=-p=1 GOMAXPROCS=2 RUN CGO_ENABLED=0 GOOS=linux go build \ -trimpath \ @@ -32,6 +33,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ -o /out/messenger ./cmd RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + FROM builder AS tests COPY --from=goose-builder /out/goose /usr/local/bin/goose ENV GOOSE_MIGRATION_DIR=/src/migrations @@ -39,6 +42,8 @@ CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 +RUN apk add --no-cache libwebp-tools=1.6.0-r0 + RUN addgroup -S messenger \ && adduser -S -D -H -u 10001 -G messenger messenger @@ -46,6 +51,8 @@ COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/me COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile +COPY --from=builder --chown=messenger:messenger /out/worker /usr/local/bin/worker + ENV PORT=8080 EXPOSE 8080 diff --git a/Makefile b/Makefile index 8c7ec355..6f2aa27c 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) GO ?= go -PORT ?= 8080 +PORT ?= 18085 BASE_URL ?= http://localhost:$(PORT) -IMAGE ?= mtusi-messenger:lab4 +IMAGE ?= mtusi-messenger:lab5 CA_FILE ?= ACTION_SCRIPT ?= scripts/restart.sh FAILOVER_SCRIPT ?= scripts/stop-one.sh @@ -15,7 +15,7 @@ TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) export PORT BASE_URL help: - @echo "up: build and deploy Swarm stack; down: stop with volumes retained; test-integration: PostgreSQL race/rollback/EXPLAIN; test-grade4: cross-replica idempotency, both failures and rolling update; check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up: build and deploy isolated lab5 stack; test-lab5: pipeline faults; benchmark: queue research; down: keep volumes; test-integration: PostgreSQL race/rollback/EXPLAIN; test-grade4: cross-replica idempotency, both failures and rolling update; check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -69,10 +69,10 @@ test-grade4: $(PYTHON) tests/own/swarm_grade4.py test-lab4: - $(PYTHON) tests/own/lab4.py + $(PYTHON) tests/own/lab5.py test-reconcile: - $(PYTHON) tests/own/reconcile.py + $(PYTHON) tests/own/lab5.py measure-uploads: $(PYTHON) scripts/measure_uploads.py @@ -94,3 +94,26 @@ test-failover: wait: $(PYTHON) scripts/swarm.py wait + +.PHONY: test-lab5 benchmark plot-research worker-stop worker-start +test-lab5: + $(PYTHON) tests/own/lab5.py + +benchmark: + $(PYTHON) scripts/benchmark_pipeline.py --base-url "$(BASE_URL)" + +plot-research: + $(PYTHON) scripts/plot_pipeline.py + +worker-stop: + $(PYTHON) scripts/swarm.py worker-stop + +worker-start: + $(PYTHON) scripts/swarm.py worker-start + +.PHONY: failed-jobs replay +failed-jobs: + $(PYTHON) scripts/swarm.py failed-jobs + +replay: + $(PYTHON) scripts/swarm.py replay --replay "$(JOB_ID)" --operator "$(OPERATOR)" --reason "$(REASON)" diff --git a/PORTING.md b/PORTING.md new file mode 100644 index 00000000..447b5208 --- /dev/null +++ b/PORTING.md @@ -0,0 +1,19 @@ +# Переход с lab4 на lab5 + +Исходный проект `mtusi2026lab4` сохранён без изменений. +Go module path сохранён для существующих импортов. +Нормативные контракты и публичные проверки lab5 восстановлены из исходного skeleton. + +Загрузка теперь возвращает `202 queued` после staging PUT и PostgreSQL COMMIT. +Старая строка ready без вариантов проходит backfill 009 и получает все три варианта. +При обновлении `make up` останавливает только API своего стека перед backfill. +Это исключает запись старым процессом после миграции. +Volumes и Docker secrets сохраняются. + +Стек, образы и секреты используют имена lab5. Внешний порт по умолчанию — 18085. +Стенд lab4 на 8080 не изменяется. +Redis хранит сессии и durable Streams в одном AOF volume. +Worker работает отдельным сервисом; API не выполняет resize. + +Новая схема, команды и проверки описаны в [REPORT.md](REPORT.md). +Старые `evidence/lab4-*` относятся к исходному проекту и не доказывают выполнение lab5. diff --git a/README.md b/README.md index 009d6997..d178c9ef 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ Отделить приём файла от тяжёлой обработки, корректно переживать повторную доставку и остановку worker. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Реализация использует Redis Streams, PostgreSQL outbox и отдельный worker. +API сохраняет оригинал в приватный MinIO staging и возвращает `202 queued`. +Worker создаёт JPEG thumbnail, JPEG crop и WebP optimized. +Запуск: `make up`; адрес: http://localhost:18085; Swagger: `/swagger/`. +Проверки: `make test`, `make test-integration`, `make test-lab5`. +Инструкции и доказательства находятся в [REPORT.md](REPORT.md). +Исследование перегрузки описано в [RESEARCH.md](RESEARCH.md). [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -52,16 +58,17 @@ make check # После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +# По умолчанию Makefile использует http://localhost:18085. make test ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +`compose.yaml` и `stack.yaml` запускают готовый стенд с закреплёнными версиями образов. Роли компонентов описаны в `contracts/infrastructure.json`. `make bootstrap` создаёт секреты в исключённом из Git каталоге `secrets/`; Swarm использует Docker secrets. Миграции выполняются перед запуском API. Все зависимости поднимаются из этого репозитория. Порядок запуска описан в [REPORT.md](REPORT.md). ```sh # Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build +./scripts/compose.sh config --quiet +make bootstrap +./scripts/compose.sh up -d --build --wait # Дождитесь /health/ready; затем make test. # Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. ``` diff --git a/REPORT.md b/REPORT.md index e98f75e9..49fa5eff 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,268 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5: очередь изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка: **5**, исследование — **трек 1, очередь под перегрузкой**. +Redis уже хранит сессии. Redis Streams добавляет очередь без второго брокера. +Исследование сравнивает три настройки concurrency/prefetch. Методика и результаты: [RESEARCH.md](RESEARCH.md). +Публичные контракты и проверки lab5 сохранены. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Установите Docker с поддержкой Swarm и Python 3.9+. +Локальные Go-проверки требуют Go 1.26.4 или новее. +Запустите стенд: -## Архитектура и решения +```sh +make up +make wait +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +API: http://localhost:18085. Swagger: http://localhost:18085/swagger/. +Стек `mtusi-lab5` не использует volumes, секреты и порт исходного стенда lab4. +Скрипт поддерживает один узел Swarm. Порты хранилищ не опубликованы. -## Доказательства критериев +Bootstrap создаёт случайные секреты в локальном каталоге `secrets/` и Docker secrets. +Каталог исключён из Git и Docker context. Скрипты не загружают `.env`. +Версии закреплены в Dockerfile/stack.yaml: -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Компонент | Версия | +| --- | --- | +| Go / Alpine | 1.26.4 / 3.23, базовые digest в Dockerfile | +| PostgreSQL | 17.6-alpine3.22 | +| Redis | 7.4.5-alpine3.21 | +| HAProxy | 3.2.6-alpine | +| MinIO | RELEASE.2025-09-07T16-13-09Z, фиксированный commit и SHA-256 | +| mc | RELEASE.2025-08-13T08-35-41Z, фиксированный commit и SHA-256 | +| libwebp-tools | 1.6.0-r0 | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Go compiler использует `-p=1` и `GOMAXPROCS=2` при сборке контейнеров. +Это ограничивает память сборки на Docker VM с 1,91GiB RAM. +Первый параллельный build получил OOM; последовательный build устранил эту ошибку. +`SKIP_STORAGE_BUILD=1 make up` повторно использует уже собранные образы MinIO/mc. +Обычный `make up` собирает все образы из репозитория. -## Эксперименты и отказы +Compose запускает те же роли с одной API-репликой: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```sh +make bootstrap +./scripts/compose.sh up -d --build --wait +make test +./scripts/compose.sh down +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Остановите один стенд перед запуском другого на том же порту. +`make down` сохраняет volumes и Docker secrets. +При обновлении `make up` останавливает API перед миграциями 005–010. +Миграция 009 переводит старые ready-оригиналы lab4 в очередь без повторной загрузки. +Старый object key временно становится staging key. +Worker удаляет этот staging только после публикации готовых вариантов. +Эта пауза исключает запись старой API-версией после backfill. +Откат миграций удаляет jobs/outbox; сохранять принятые задачи при downgrade нельзя. -## Ограничения и следующие шаги +## Путь принятой загрузки -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +1. API проверяет сессию, членство в чате, имя, bytes, MIME и пиксели. +2. API сохраняет проверенный оригинал в приватный `staging/`. +3. PostgreSQL одной транзакцией сохраняет attachment, job и outbox. +4. API возвращает `202` со снимком `queued` после COMMIT. +5. Publisher читает outbox через `FOR UPDATE SKIP LOCKED` и делает XADD. +6. Worker получает сообщение через XREADGROUP и арендует job в PostgreSQL. +7. Renderer создаёт варианты в отдельном процессе с deadline. +8. Worker сохраняет оригинал и три варианта по постоянным versioned keys. +9. PostgreSQL атомарно публикует ready и метаданные всех вариантов. +10. Worker подтверждает запись через XACK и удаляет подтверждённую запись XDEL. + +API не выполняет resize. Проверка входного изображения в API сохраняет требования lab4. +Она включает полный decode для обнаружения повреждений, но не создаёт варианты. +Каждая API-реплика проверяет не более одного upload одновременно. +Локальный spool ограничен 10MiB и удаляется после staging PUT или ошибки. + +API publisher работает независимо от worker. Worker также может опубликовать ожидающий outbox. +Row locks исключают конкурентную публикацию одной outbox-строки. +Разрыв между XADD и PostgreSQL COMMIT может создать дубль; потеря принятой задачи не возникает. +Подтверждение XADD означает durable запись Redis с `appendfsync always`. +XACK означает устойчивый результат обработки либо устойчивую запись следующего состояния. +Это at-least-once доставка с идемпотентным эффектом, а не exactly-once доставка. + +Redis хранит stream `attachments:jobs:v1`, группа — `image-workers-v1`. +Одно поле stream `job` содержит JSON по `contracts/jobs.schema.json`. +Поля включают UUID job/attachment, version, attempt, type, schema_version и created_at. +Сообщение не содержит bytes, credentials, filename, URL или S3 key. +Worker берёт доверенные storage metadata из PostgreSQL. + +## Состояния, retry и защита результата + +PostgreSQL — источник истины состояний queued/processing/ready/failed. +Queued и processing можно прикрепить к сообщению своего чата. +Content до ready и после failed возвращает 409. Метаданные доступны во всех состояниях. +Failed нельзя прикрепить заново. Ранее подтверждённые сообщения и их idempotency replay сохраняются. + +Claim увеличивает attempts и fence, задаёт lease_until. +Фиксация результата требует текущий fence, pipeline_version и ещё действующий lease. +Она также требует ровно одну изменённую строку attachment нужной версии и состояния. +Старый процесс не публикует ready после аренды новой попыткой. +Объекты используют ключ `original//v/`. +Имена эффектов: original, thumbnail, crop, optimized. +Повтор той же версии пишет те же байты в те же четыре ключа. +Новая версия получает другой prefix. Стандартный pipeline_version равен 1. + +XAUTOCLAIM возвращает pending-доставки умерших consumers после lease. +Истёкший processing возвращается в queued через устойчивую транзакцию и новый outbox. +Attempts учитывают управляемые попытки обработки; простой дубль ready не увеличивает attempts. +Перед ack дубля ready проверяются метаданные и наличие четырёх S3-объектов нужного размера. +Дубль не изменяет updated_at, варианты или готовый результат. + +Временные ошибки хранилищ, timeout и worker crash используют ограниченный retry. +Постоянные ошибки изображения и исчезнувший staging дают failed сразу. +Backoff после попытки n: `2^min(n,5)` секунд плюс случайные 0–499ms. +По умолчанию разрешено пять попыток. Финальный public error_code безопасен и не содержит внутренних данных. +Реестр failed хранится в attachment_jobs; последняя причина сохраняется в last_error. + +```sh +make failed-jobs +make replay JOB_ID= OPERATOR= REASON='Причина повторной обработки' +``` + +Replay принимает только failed с доступным staging и свободным admission slot. +Он сохраняет job_id и versioned effect identity, сбрасывает счётчик попыток и увеличивает fence. +Транзакция записывает operator, reason, предыдущую ошибку и время в attachment_replays. +Replay ready или ещё активной работы отклоняется. После истечения retention replay отклоняется. + +## Изображения и пределы ресурсов + +Thumbnail — JPEG quality85, вписан в 256×256 без увеличения, размеры округляются вниз. +Crop — центрированный квадрат JPEG128×128 quality85, малые исходники увеличиваются. +Optimized — WebP quality80, method4, вписан в 1280×1280 без увеличения. +JPEG/PNG EXIF orientation1–8 применяется до изменения размеров. +Alpha компонуется на белый. Варианты не сохраняют EXIF; cwebp получает `-metadata none`. +Уменьшение каждого маленького файла не обещается. Оригинал сохраняется побайтно. + +API ограничивает bytes10MiB и 40 миллионов пикселей до полного decode. +Worker повторно проверяет размер, SHA-256, DecodeConfig и pixel budget. +Resize/decode/cwebp работают в дочернем renderer; CommandContext ограничивает весь процесс. +Renderer создаёт отдельную process group. Deadline убивает renderer и потомков, включая cwebp. +Общий pixel semaphore ограничивает одновременно распакованные исходники 40Mpx на worker service. +Orientation использует представление исходника, без второй копии полного изображения. +Renderer получает GOMEMLIMIT384MiB; worker service ограничен cgroup1536MiB и1CPU. +GOMEMLIMIT является мягким Go-лимитом; жёсткий предел задаёт cgroup всей службы. + +| Настройка | По умолчанию | Назначение | +| --- | --- | --- | +| WORKER_CONCURRENCY | 2 | Одновременные обработки, максимум8 | +| WORKER_PREFETCH | 4 | Ёмкость локальной очереди, максимум32 | +| WORKER_TIMEOUT_SECONDS | 25 | Вся попытка: admission пикселей, download, renderer и PUT | +| WORKER_LEASE_SECONDS | 40 | Lease; минимум timeout+5s | +| WORKER_PIXEL_BUDGET | 40000000 | Общий pixel budget, максимум80000000 | +| WORKER_MAX_ATTEMPTS | 5 | Число попыток новой job, задаётся API при upload | +| MAX_OUTSTANDING_JOBS | 200 | Максимум queued+processing на весь стенд | + +Prefetch ограничивает ожидающую локальную очередь. Активные процессы добавляют concurrency к числу полученных сообщений. +Настройки bounded; число принятых незавершённых jobs ограничено отдельно. +Admission использует общий PostgreSQL advisory lock и count queued/processing. +Этот lock также защищает replay. Превышение возвращает 429 с Retry-After. +Проверка и вставка выполняются в одной транзакции, поэтому две API-реплики не превышают лимит. +Lock удерживается во время staging PUT. Это ограничивает throughput upload и измеряется в исследовании. + +## Retention и согласованность S3 + +Очистка выполняется каждые пять секунд, не более100 metadata rows за проход. +Ready staging удаляется после устойчивой публикации всех вариантов. +Failed staging и частичные конечные объекты сохраняются семь дней для расследования и replay. +После семи дней cleanup удаляет bytes и ставит storage_cleaned_at, но сохраняет metadata/job/audit. +Replay и cleanup блокируют одну metadata row; очистка не удаляет текущую работу. +Старый stage без metadata имеет grace90s и общий с upload advisory lock. +После PUT без COMMIT API возвращает ошибку; такой stage удаляется после grace. +Готовые конечные вложения сохраняются бессрочно, включая ещё не привязанные к сообщению. +Их staging очищается одинаково. Автоматическое удаление готовых конечных вложений не реализовано. + +Reconciler lab4 сохранён: `make reconcile`, `make reconcile-apply`. +Он защищает текущий versioned prefix и staging_key, включая legacy staging в original-prefix. +Отсутствующий ready-оригинал атомарно переводит attachment и job в failed. +Reconciler не удаляет ссылки из подтверждённых сообщений. +Reconciler контролирует original; внешняя утрата отдельного варианта требует ручного восстановления. +Проверка дубля пишет безопасный retry-log, если готовый эффект недоступен. +Автоматический ремонт внешних удалений вариантов не реализован. + +## Проверки и исследование + +Подготовьте зависимости проверок: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-research.txt +make check validate +make test-own +make test-integration +make test +make test-lab5 +make test-persistence +make test-failover +make benchmark +make plot-research +``` + +Go-тесты проверяют геометрию, EXIF1–8, alpha, повреждения, MIME, concurrency и транзакции. +Интеграционный тест проверяет stale/expired fence и защиту более новой pipeline_version. +Fault suite внедряет ошибку после staging, разрыв COMMIT/publish и два broker restart. +Он останавливает worker после S3 effects и после result COMMIT перед ack. +Проверяются повторная доставка, lease recovery, bounded retry, permanent failure и audit replay. +Отдельный renderer_timeout блокирует настоящий cwebp на FIFO и проверяет уничтожение потомков. +Очистка проверяется для ready, unconfirmed и expired failed staging. +Все пользователи и изображения синтетические. Токены и секреты в evidence не сохраняются. +Запускайте fault, persistence и benchmark последовательно на учебном стенде. + +Исследование трека1 использует реальные свежие измерения, а не результаты исходного проекта. +Пары concurrency/prefetch: (1,1), (2,4), (4,8); размеры PNG512² и1536². +Отдельный опыт проверяет предел admission при остановленном worker и последующее восстановление. +Raw данные, графики, ограничения методики и объяснение для защиты находятся в [RESEARCH.md](RESEARCH.md). +Исследование завершено: все 120 измеряемых загрузок стали ready, шесть сочетаний настроек и размеров проверены. +Пара concurrency=2, prefetch=4 дала лучший throughput для обоих размеров: 4,013 и 0,851 изображения/с. +При остановленном worker лимит 8 принял 8 из 16 загрузок; остальные получили 429. +После запуска все 8 заданий завершились, очередь опустела, новая загрузка прошла. +Текстовое сообщение сохранилось при полной очереди. Исходные настройки восстановлены. +Итог `PASS` и наблюдения: `evidence/lab5-research.json`; таблица: `evidence/lab5-research.csv`. +Исторические evidence/lab4-* не считаются доказательством lab5. + +`make test` прошёл на новом API lab5. Журнал: `evidence/lab5-smoke.txt`. +Host race-тесты и контейнерные интеграционные тесты прошли. +Интеграционный журнал: `evidence/lab5-integration.txt`; исходная ошибка fixture сохранена отдельно. +Прежний fixture удалял migration journal из уже обновлённой схемы. +Теперь он сначала возвращает физическую схему lab4, затем проверяет её импорт. + +Все обязательные fault cases проверены на реальных PostgreSQL, Redis Streams и MinIO. +Первый полный вызов подтвердил основные durability/crash/retry сценарии и выявил гонку тестового stop helper. +Desired replicas=0 ещё не означает остановку старого consumer-контейнера. +Helper и `make worker-stop` теперь ожидают фактическое отсутствие работающих контейнеров. +Поздние независимые cases повторены через `--tail-only` с новыми пользователями и изображениями. +Повтор завершился exit0; прежние успешные cases и время продолжения сохранены в JSON. +Исходный полный вызов завершился exit1 и не объявляется успешным полным прогоном. +Журналы: `evidence/lab5-faults-main.txt`, `evidence/lab5-faults-tail.txt`. +Все case-level результаты находятся в `evidence/lab5-faults.json`. + +Отдельный initial fault обнаружил старый worker при обновлении образа с прежним tag. +`make up` теперь принудительно обновляет worker вместе с API. +После исправления фактические JPEG/PNG EXIF проверки прошли. +Статические проверки и команды воспроизведения собраны в `evidence/lab5-validation.txt`. +`make test-persistence` прошёл после перезапуска компонентов собственного стенда. +Подтверждённые данные и существующая сессия сохранились: `evidence/lab5-persistence.txt`. +`make test-failover` прошёл при остановке одной из двух API-реплик. +Зафиксированы 123 успешные записи, одна неудачная попытка и восстановление за 0,696s. +Подтверждённые данные и сессия сохранились: `evidence/lab5-failover.txt`. + +## Соответствие критериям + +| Уровень | Критерий | Реализация и доказательство | +| --- | --- | --- | +| 3 | Возможности lab4 и переход upload на 202 | `PORTING.md`, публичный smoke, persistence/failover | +| 3 | Устойчивый приём до ответа | MinIO staging + PostgreSQL attachment/job/outbox; cases `stage_before_commit`, `commit_publish_and_restarts` | +| 3 | Отдельный broker/worker, durable at-least-once | Redis Streams/AOF; два broker restart, worker stop/start | +| 3 | Четыре состояния, оригинал и thumbnail | Контракт API; smoke, `small_alpha_exif`, `bounded_retry_replay` | +| 3 | Ограниченные повторы и единый результат | `max_attempts`, versioned keys; cases `crash_after_objects`, `commit_before_ack_and_duplicate` | +| 4 | Crop/WebP, качество и ограничения | Renderer, pixel budget, process deadline; EXIF/alpha/geometry tests и `renderer_process_group_timeout` | +| 4 | Восстановление разрыва COMMIT/publish | Transactional outbox; case `commit_publish_and_restarts` | +| 4 | Backoff/jitter, permanent failure, failed registry/replay | Cases `bounded_retry_replay`, `permanent_stage_error`; audit replay | +| 4 | Crash после S3, lease/fence, bounded concurrency/prefetch | Crash cases, `TestFenceIntegration`, конфигурация worker | +| 4 | Безопасная очистка staging | Case `staging_cleanup`; ready/failed/unconfirmed retention | +| 5 | Трек 1: настройки, размеры, backpressure и восстановление | Методика, raw данные, графики и ограничения в `RESEARCH.md` | + +Источники case-level результатов: `evidence/lab5-faults.json` и журналы split-run выше. diff --git a/RESEARCH.md b/RESEARCH.md new file mode 100644 index 00000000..742a681b --- /dev/null +++ b/RESEARCH.md @@ -0,0 +1,185 @@ +# Исследование: очередь под перегрузкой + +Выбран трек 1 лабораторной 5. Исследование сравнивает скорость, задержку и память при разных настройках обработки. +Отдельный сценарий проверяет ограничение очереди и восстановление после перегрузки. + +## Гипотеза + +Рост числа параллельных обработок уменьшает ожидание, пока хватает процессора. +После насыщения процессора дополнительные обработки увеличивают память и конкурируют за вычисления. +Ограничение очереди сохраняет конечное число принятых заданий, когда worker остановлен. + +## Что означают настройки + +- `WORKER_CONCURRENCY` ограничивает число параллельных обработок внутри одного worker. +- `WORKER_PREFETCH` ограничивает буфер заданий перед обработкой. +- `WORKER_PIXEL_BUDGET` ограничивает суммарное число пикселей у активных обработок. +- `MAX_OUTSTANDING_JOBS` ограничивает число заданий со статусом `queued` или `processing` в PostgreSQL. + +Redis Streams хранит сообщения до подтверждения. PostgreSQL хранит состояние задания и намерение публикации. +Сообщение Redis содержит идентификаторы. Байты изображения остаются в S3. +Полная очередь приводит к HTTP `429`. Текстовые сообщения продолжают работать. + +## Методика + +Стенд включает один worker, две API-реплики, PostgreSQL, Redis и MinIO. +Ограничения CPU и памяти worker сохраняются между запусками. +Исходные изображения — PNG с псевдослучайными RGB-пикселями. Генератор использует фиксированное начальное значение. +SHA-256 каждого изображения записывается в результат. + +Скрипт сравнивает шесть запусков: + +| Размер PNG | Concurrency | Prefetch | Число изображений | +| --- | ---: | ---: | ---: | +| 512×512 | 1 | 1 | 20 | +| 512×512 | 2 | 4 | 20 | +| 512×512 | 4 | 8 | 20 | +| 1536×1536 | 1 | 1 | 20 | +| 1536×1536 | 2 | 4 | 20 | +| 1536×1536 | 4 | 8 | 20 | + +Перед каждым запуском worker завершает все задания. Одно изображение прогревает pipeline после изменения настройки. +Измерение начинается после прогрева. Восемь HTTP-потоков отправляют двадцать изображений. +Скрипт опрашивает метаданные до `ready` для всех принятых вложений. +После измерения скрипт проверяет оригинал и контрольные суммы трёх вариантов у одного вложения. + +## Метрики + +| Метрика | Определение | +| --- | --- | +| Throughput | Число готовых изображений / время всего пакета, включая upload и опрос | +| p95 до ready | 95-й процентиль времени от начала POST до первого ответа `ready` | +| p95 после 202 | 95-й процентиль времени от ответа `202` до первого ответа `ready` | +| Queue lag | Возраст старейшего задания `queued` из текущего чата; `processing` исключается | +| RSS | Сумма `VmRSS` worker, процессов обработки и `cwebp` | +| Размер очереди | Число заданий `queued` и `processing` в PostgreSQL | + +Процентиль рассчитывается методом ближайшего ранга. Для двадцати изображений p95 соответствует девятнадцатому значению после сортировки. +Скрипт снимает RSS и queue lag между опросами API. Пауза между циклами составляет минимум 250 мс. +Фактические интервалы содержатся в JSON. Команды Docker и запросы API увеличивают этот интервал. + +## Результаты измерений + +Дата измерений: 10 ноября 2026 года. Архитектура — arm64, Docker Engine — 29.5.2. +Docker VM располагает двумя CPU и 1,91 GiB RAM. +Worker ограничен одним CPU и 1536 MiB памяти. Эти ограничения одинаковы во всех шести запусках. +Размеры входных файлов составили 787252 и 7081647 байт. +Все 120 измеряемых загрузок вернули `202` и завершились статусом `ready`. + +| PNG | Concurrency / prefetch | Throughput, изобр./с | p95 до ready, с | Queue lag max, с | RSS max, МиБ | +| --- | ---: | ---: | ---: | ---: | ---: | +| 512×512 | 1/1 | 3.316 | 4.76 | 4.08 | 32.2 | +| 512×512 | 2/4 | 4.013 | 4.33 | 3.17 | 54.2 | +| 512×512 | 4/8 | 3.903 | 4.45 | 2.67 | 90.3 | +| 1536×1536 | 1/1 | 0.774 | 22.79 | 21.48 | 74.2 | +| 1536×1536 | 2/4 | 0.851 | 21.47 | 18.20 | 134.3 | +| 1536×1536 | 4/8 | 0.847 | 20.74 | 16.60 | 224.2 | + +![Результаты шести измерений](evidence/lab5-research.svg) + +PNG-версия графика: [lab5-research.png](evidence/lab5-research.png). +Точные значения, дополнительные метрики и исходные наблюдения доступны в JSON и CSV. + +Для этого стенда рекомендована пара **concurrency=2, prefetch=4**. Она совпадает с настройкой по умолчанию. +Она дала максимальный измеренный throughput для обоих размеров. +Относительно 1/1 throughput вырос примерно на 21% для маленьких изображений и на 10% для крупных. +Пиковый RSS при этом вырос с 32,2 до 54,2 МиБ и с 74,2 до 134,3 МиБ. + +Переход от 2/4 к 4/8 не увеличил throughput. +Для крупных изображений p95 улучшился примерно на 3,4%, а RSS вырос примерно на 67%. +Для маленьких изображений выросли и RSS, и p95. +Выбор 1/1 уменьшает расход памяти ценой большей задержки и меньшей скорости. + +Результат согласуется с гипотезой об ограниченной пользе параллелизма при одном CPU. +Он не доказывает причину насыщения: upload, S3 и PostgreSQL также входят в измерение. +Меньший queue lag при 4/8 не означает пропорционального ускорения pipeline. +Метрика исключает задания `processing`, число которых может расти вместе с concurrency. +Разницу в несколько процентов следует проверить повторными сериями перед изменением рабочих настроек. + +## Перегрузка и восстановление + +Скрипт останавливает worker и устанавливает лимит восемь заданий на обеих API-репликах. +Затем шестнадцать запросов конкурируют за этот общий лимит. +Проверка требует восемь ответов `202`, восемь ответов `429` и восемь заданий в PostgreSQL. +Заголовки `X-Instance-Id` подтверждают участие обеих API-реплик. + +После запуска worker все восемь принятых заданий завершаются. +Скрипт проверяет пустую очередь и новый успешный upload. +Скрипт восстанавливает исходные настройки после эксперимента, в том числе при ошибке проверки. + +Фактический результат: + +| Проверка | Результат | +| --- | --- | +| Остановленный worker, лимит 8, предложено 16 | 8 ответов `202`, 8 ответов `429` | +| Участие API | Подтверждены две разные реплики | +| Незавершённые задания при остановленном worker | Ровно 8 | +| Текстовое сообщение при полной очереди | Сохранено успешно | +| Принятые задания после запуска worker | Все 8 стали `ready` | +| Незавершённые задания после восстановления | 0 | +| Следующая загрузка после освобождения очереди | `202`, затем `ready` | + +Счётчик завершения проверяется после ожидания готовности worker. +Часть обработки проходит во время этого ожидания, поэтому отдельное поле drain не измеряет полное время восстановления. +Опыт подтверждает общий лимит между репликами и сохранность принятой работы после `202`. +Эксперимент завершился с `PASS`. Исходные настройки worker и API восстановлены и проверены отдельно. + +## Повторение + +Запустите стенд: + +```sh +make up +make test +``` + +Остановите другие нагрузочные проверки. Запустите исследование: + +```sh +make benchmark +``` + +Установите зависимости для графика: + +```sh +.venv/bin/python -m pip install -r requirements-research.txt +``` + +Создайте график: + +```sh +make plot-research +``` + +Скрипт сохраняет исходные наблюдения в `evidence/lab5-research.json`. +Таблица находится в `evidence/lab5-research.csv`. Графики сохраняются в SVG и PNG рядом с таблицей. +JSON содержит версию Docker, число CPU, память, образ worker и ограничения контейнера. + +## Ограничения + +Измерения относятся к одному локальному стенду. Пакет из двадцати изображений не определяет производственный SLA. +В каждом запуске есть прогрев, но независимых повторений для доверительного интервала нет. +Concurrency и prefetch меняются вместе. Результат сравнивает пары настроек и не изолирует влияние каждого параметра. + +HTTP upload, проверка изображения, S3 и PostgreSQL также влияют на throughput. +Глобальная блокировка при приёме задания сериализует участок записи staging и регистрации задания. +Увеличение concurrency не устраняет это ограничение API. +Admission ограничивает незавершённую работу. Concurrency и бюджет пикселей ограничивают память обработки. +Этот лимит не ограничивает общий размер готовых вложений на диске. + +Опрос добавляет задержку к наблюдаемому времени до `ready`. +Выборка RSS может пропустить короткий пик. Общие страницы памяти могут учитываться несколько раз. +Небольшая часть RSS относится к самой команде наблюдения внутри контейнера. +Значение RSS не равно использованию памяти cgroup с файловым кешем. + +## Объяснение для защиты + +1. API возвращает `202` после сохранения файла и задания. Обработка изображения продолжается отдельно. +2. Redis хранит очередь доставки. PostgreSQL хранит состояние и восстанавливает публикацию через outbox. +3. Concurrency задаёт число одновременных обработок. Prefetch задаёт запас заданий перед обработкой. +4. Задержка растёт, когда задания поступают быстрее обработки. Бесконечная очередь увеличивает ожидание и расход ресурсов. +5. Лимит очереди возвращает `429` для новой работы. Работа после уже выданного `202` сохраняется. +6. После запуска worker очередь завершается. Повторный upload проходит без изменения принятого ранее задания. +7. Подтверждение Redis происходит после устойчивого результата. Повтор доставки безопасен благодаря состоянию задания и ключам вариантов. + +Это доставка at least once. Повтор доставки возможен, поэтому результат защищён от повторного применения. diff --git a/cmd/main.go b/cmd/main.go index 6f1f721d..f3be9581 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -63,7 +63,15 @@ func main() { if err != nil { log.Fatal("invalid S3 config") } - handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachment.New(pool, objectStore)) + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + attachments := attachment.NewAsync(pool, objectStore, options) + background, stopPublisher := context.WithCancel(context.Background()) + defer stopPublisher() + go attachments.RunPublisher(background, redisClient) + handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachments) port := cfg.Port e := handler.Router() e.Server.ReadHeaderTimeout = 5 * time.Second diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..2d5de868 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "render" { + if len(os.Args) != 5 { + os.Exit(2) + } + if err := attachment.Render(context.Background(), os.Args[2], os.Args[3], os.Args[4]); err != nil { + if errors.Is(err, attachment.ErrInvalidImage) { + os.Exit(2) + } + os.Exit(1) + } + return + } + failed := flag.Bool("failed", false, "list failed job registry") + replay := flag.String("replay", "", "failed job UUID to replay") + operator := flag.String("operator", "", "operator identity") + reason := flag.String("reason", "", "replay reason") + flag.Parse() + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["timezone"] = "UTC" + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.MaxConns = int32(options.Concurrency + 4) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("DB configuration failed") + } + defer pool.Close() + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("S3 configuration failed") + } + service := attachment.NewAsync(pool, store, options) + if *failed { + rows, e := pool.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts,coalesce(last_error,''),updated_at FROM attachment_jobs WHERE state='failed' ORDER BY updated_at DESC LIMIT 100`) + if e != nil { + log.Fatal("cannot query failed jobs") + } + defer rows.Close() + records := []map[string]interface{}{} + for rows.Next() { + var id, att, code string + var attempt, max int + var updated time.Time + if e = rows.Scan(&id, &att, &attempt, &max, &code, &updated); e != nil { + log.Fatal("invalid failed job") + } + records = append(records, map[string]interface{}{"job_id": id, "attachment_id": att, "attempts": attempt, "max_attempts": max, "error_code": code, "updated_at": updated.UTC()}) + } + if rows.Err() != nil { + log.Fatal("failed jobs query interrupted") + } + json.NewEncoder(os.Stdout).Encode(records) + return + } + if *replay != "" { + operation, stop := context.WithTimeout(ctx, 10*time.Second) + defer stop() + if err = service.Replay(operation, *replay, *operator, *reason); err != nil { + log.Fatal(err) + } + log.Print("Replay committed with audit record") + return + } + r := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr, DialTimeout: 2 * time.Second, ReadTimeout: 2 * time.Second, WriteTimeout: 2 * time.Second, MaxRetries: -1, ContextTimeoutEnabled: true}) + defer r.Close() + log.Printf("Worker started: concurrency=%d prefetch=%d timeout=%s lease=%s pixel_budget=%d", options.Concurrency, options.Prefetch, options.Timeout, options.Lease, options.PixelBudget) + for ctx.Err() == nil { + err = service.RunWorker(ctx, r) + if err != nil { + log.Print("worker broker connection will retry") + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } +} diff --git a/compose.yaml b/compose.yaml index 4c221cd3..7d43834e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -name: mtusi2026lab4 +name: mtusi2026lab5 services: postgres: image: postgres:17.6-alpine3.22 @@ -29,7 +29,7 @@ services: timeout: 2s retries: 30 cpus: 0.5 - mem_limit: 128m + mem_limit: 384m minio: image: mtusi-minio:RELEASE.2025-09-07T16-13-09Z build: @@ -66,7 +66,7 @@ services: cpus: 0.5 mem_limit: 512m migrations: - image: mtusi-migrations:lab4 + image: mtusi-migrations:lab5 build: context: . target: migrations @@ -83,7 +83,7 @@ services: mem_limit: 128m api: build: . - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 environment: &api_environment POSTGRES_HOST: postgres POSTGRES_USER: messenger @@ -107,7 +107,7 @@ services: - s3_access_key - s3_secret_key ports: - - '127.0.0.1:${PORT:-8080}:8080' + - '127.0.0.1:${PORT:-18085}:8080' depends_on: postgres: condition: service_healthy @@ -120,8 +120,29 @@ services: cpus: 1.0 mem_limit: 1536m stop_grace_period: 10s + worker: + image: mtusi-messenger:lab5 + build: . + entrypoint: [/usr/local/bin/worker] + environment: + <<: *api_environment + WORKER_CONCURRENCY: '2' + WORKER_PREFETCH: '4' + WORKER_MAX_ATTEMPTS: '5' + WORKER_TIMEOUT_SECONDS: '25' + WORKER_LEASE_SECONDS: '40' + WORKER_PIXEL_BUDGET: '40000000' + secrets: *api_secrets + healthcheck: {disable: true} + depends_on: + migrations: {condition: service_completed_successfully} + s3_bootstrap: {condition: service_completed_successfully} + redis: {condition: service_healthy} + cpus: 1.0 + mem_limit: 1536m + stop_grace_period: 10s tests: - image: mtusi-messenger-tests:lab4 + image: mtusi-messenger-tests:lab5 profiles: [test] build: context: . diff --git a/contracts/infrastructure.json b/contracts/infrastructure.json index dd2edf2d..a4cfcb7b 100644 --- a/contracts/infrastructure.json +++ b/contracts/infrastructure.json @@ -1,5 +1,5 @@ { - "lab": 4, + "lab": 5, "kind": "requirements, not runnable deployment", "required_for_grade_3": [ { @@ -27,6 +27,22 @@ ], "private_bucket": true, "persistent_volume": true + }, + { + "role": "broker", + "choose_one": [ + "RabbitMQ", + "Redis Streams", + "Valkey Streams", + "NATS JetStream", + "documented durable equivalent" + ], + "persistent_volume": true, + "delivery": "at-least-once" + }, + { + "role": "worker", + "separate_process": true } ], "optional_components": [ diff --git a/contracts/jobs.schema.json b/contracts/jobs.schema.json new file mode 100644 index 00000000..2e9aa484 --- /dev/null +++ b/contracts/jobs.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mtusi:messenger:attachment-job:v1", + "title": "Durable attachment processing job v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "job_id": { + "type": "string", + "format": "uuid" + }, + "type": { + "const": "attachment.process.v1" + }, + "attachment_id": { + "type": "string", + "format": "uuid" + }, + "pipeline_version": { + "type": "integer", + "minimum": 1 + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "traceparent": { + "type": "string", + "pattern": "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" + }, + "tracestate": { + "type": "string", + "maxLength": 512 + } + }, + "examples": [ + { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z" + } + ] +} diff --git a/contracts/openapi.json b/contracts/openapi.json index 138f07c0..4face909 100644 --- a/contracts/openapi.json +++ b/contracts/openapi.json @@ -1,8 +1,8 @@ { "openapi": "3.1.0", "info": { - "title": "MTUSI Messenger — лабораторная 4", - "version": "1.4.0", + "title": "MTUSI Messenger — лабораторная 5", + "version": "1.5.0", "description": "Нормативный минимум на 3. Семантика: contracts/README.md. Расширения на 4/5 не должны менять этот API. Наличие кода ошибки в схеме не требует реализовать факультативную причину его выдачи." }, "servers": [ @@ -2151,7 +2151,7 @@ "operationId": "uploadAttachment", "summary": "Принять один PNG/JPEG, до 10 MiB", "responses": { - "201": { + "202": { "description": "Успех", "headers": { "X-Request-Id": { @@ -2936,7 +2936,10 @@ "schema": { "type": "string", "enum": [ - "original" + "original", + "thumbnail", + "crop", + "optimized" ], "default": "original" } @@ -3461,7 +3464,10 @@ "status": { "type": "string", "enum": [ - "ready" + "queued", + "processing", + "ready", + "failed" ] }, "variants": { diff --git a/evidence/lab5-deploy.txt b/evidence/lab5-deploy.txt new file mode 100644 index 00000000..e3d056c2 --- /dev/null +++ b/evidence/lab5-deploy.txt @@ -0,0 +1,176 @@ +.venv/bin/python scripts/swarm.py up +vphhh79vd5ti8501q6m9lzulj +Local secrets are ready. +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/39 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/39 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/39 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/39 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/39 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/39 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/39 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/39 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/39 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/39 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/39 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/39 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/39 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/39 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/39 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/39 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/39 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/39 : COPY internal ./internal + ---> Using cache + ---> 420381f245d1 +Step 19/39 : COPY migrations ./migrations + ---> Using cache + ---> 244708889b45 +Step 20/39 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Using cache + ---> a86cf22e4c7d +Step 21/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> e1ea9cbf840a +Step 22/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> 4669d728553f +Step 23/39 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Using cache + ---> fe51ba375215 +Step 24/39 : FROM builder AS tests + ---> fe51ba375215 +Step 25/39 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 3ef73620b0a6 +Step 26/39 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> 43905e46829a +Step 27/39 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> 037b4f43fd3c +Step 28/39 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 29/39 : RUN apk add --no-cache libwebp-tools=1.6.0-r0 + ---> Using cache + ---> 3a741cc26d52 +Step 30/39 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 444b794708dc +Step 31/39 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> 32fc6da7172f +Step 32/39 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> 49b1f874a5e0 +Step 33/39 : COPY --from=builder --chown=messenger:messenger /out/worker /usr/local/bin/worker + ---> f7ab6619c5c8 +Step 34/39 : ENV PORT=8080 + ---> Running in 0f8f2d74139c + ---> Removed intermediate container 0f8f2d74139c + ---> ab0764165606 +Step 35/39 : EXPOSE 8080 + ---> Running in 082ca714270a + ---> Removed intermediate container 082ca714270a + ---> 9a16ff1522d8 +Step 36/39 : USER messenger:messenger + ---> Running in 2fa42c5d86db + ---> Removed intermediate container 2fa42c5d86db + ---> 50c7d9a72042 +Step 37/39 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Running in 8b938c89cf6d + ---> Removed intermediate container 8b938c89cf6d + ---> 74607fcd70f9 +Step 38/39 : STOPSIGNAL SIGTERM + ---> Running in e747d914802f + ---> Removed intermediate container e747d914802f + ---> 95748bec26d8 +Step 39/39 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Running in b502e1c9cff0 + ---> Removed intermediate container b502e1c9cff0 + ---> 66e3c9bc004e +Successfully built 66e3c9bc004e +Successfully tagged mtusi-messenger:lab5 +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/9 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/9 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/9 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/9 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/9 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/9 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/9 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/9 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/9 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Successfully built adb6e8d9b58a +Successfully tagged mtusi-migrations:lab5 +mtusi-lab5_api scaled to 0 +9mm30n5f5blu6kdfvhvartwr0 +mtusi-lab5_migrations.0.1hb6hxpeyw5j@colima | 2026/11/10 18:25:49 goose: no migrations to run. current version: 10 +mtusi-lab5_migrations +Since --detach=false was not specified, tasks will be created in the background. +In a future release, --detach=false will become the default. +Updating service mtusi-lab5_lb (id: tnlj2tzpx91wj066wqi3y5k44) +Updating service mtusi-lab5_postgres (id: f4xn3ammg3isejqsm9cqng6yo) +Updating service mtusi-lab5_redis (id: kz94wxnzhqwettqixbya643dl) +Updating service mtusi-lab5_minio (id: oaf1v50i58o60qnbk5flk0k59) +Updating service mtusi-lab5_api (id: oey97yu6vv15p4dn10pptf3b8) +Updating service mtusi-lab5_worker (id: lctq7ul45bu47a6vjbvt6xgeo) +hs7eigkxh4jah526bbx4nlt0t +mtusi-lab5_s3-bootstrap.0.veoz00w0hdfq@colima | PASS: private bucket and restricted API account configured. +mtusi-lab5_s3-bootstrap +mtusi-lab5_api scaled to 2 +mtusi-lab5_api +PASS: rolling update completed +PASS: Swarm stack ready at http://localhost:18085 diff --git a/evidence/lab5-failover.txt b/evidence/lab5-failover.txt new file mode 100644 index 00000000..b3f3d790 --- /dev/null +++ b/evidence/lab5-failover.txt @@ -0,0 +1,4 @@ +.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 1, "first_recovery_seconds": 0.696} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-faults-initial.txt b/evidence/lab5-faults-initial.txt new file mode 100644 index 00000000..757ddf02 --- /dev/null +++ b/evidence/lab5-faults-initial.txt @@ -0,0 +1,12 @@ +.venv/bin/python tests/own/lab5.py +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 302, in + main() + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 156, in main + variant_check(c,bob,item,body,shape) + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 101, in variant_check + check((variants['thumbnail']['width'],variants['thumbnail']['height'])==expected_thumb,'thumbnail geometry') + File "/Users/dkhakimovig/mtusi2026lab5/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: thumbnail geometry +make: *** [test-lab5] Error 1 diff --git a/evidence/lab5-faults-main.txt b/evidence/lab5-faults-main.txt new file mode 100644 index 00000000..34926db2 --- /dev/null +++ b/evidence/lab5-faults-main.txt @@ -0,0 +1,17 @@ +.venv/bin/python tests/own/lab5.py +PASS: input/ACL, three actual formats, geometry, EXIF, alpha +PASS: COMMIT/publish gap, queued link, API/broker restarts, worker recovery +PASS: crash after objects, expired lease/fencing, same four keys +PASS: COMMIT before ack, duplicate no-op and ready replay rejection +PASS: bounded retry/backoff, failed registry, safe audited replay +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 310, in + main() + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 277, in main + failed=wait_state(item,'failed') + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 100, in wait_state + return until(lambda: job(att), lambda j: j['state']==state, timeout) + File "/Users/dkhakimovig/mtusi2026lab5/tests/own/lab5.py", line 39, in until + raise AssertionError('condition deadline exceeded; last='+str(last)) +AssertionError: condition deadline exceeded; last={'id': '4fc5ab5e-3e10-4e78-a6ea-9599d6f941be', 'attachment_id': '7122d895-5b38-480f-8238-8df26958810f', 'pipeline_version': 1, 'state': 'ready', 'attempts': 1, 'max_attempts': 2, 'available_at': '2026-11-10T18:47:28.488754+00:00', 'lease_until': None, 'fence': 1, 'last_error': None, 'created_at': '2026-11-10T18:47:28.33972+00:00', 'updated_at': '2026-11-10T18:47:28.488935+00:00'} +make: *** [test-lab5] Error 1 diff --git a/evidence/lab5-faults-tail.txt b/evidence/lab5-faults-tail.txt new file mode 100644 index 00000000..474922c1 --- /dev/null +++ b/evidence/lab5-faults-tail.txt @@ -0,0 +1,2 @@ +PASS: input/ACL, three actual formats, geometry, EXIF, alpha +PASS: all lab5 fault, retry, replay, durability, rendering and cleanup checks diff --git a/evidence/lab5-faults.json b/evidence/lab5-faults.json new file mode 100644 index 00000000..2fba181a --- /dev/null +++ b/evidence/lab5-faults.json @@ -0,0 +1,86 @@ +{ + "stack": "mtusi-lab5", + "cases": { + "input_and_acl": [ + { + "name": "oversized", + "status": 413 + }, + { + "name": "damaged", + "status": 415 + }, + { + "name": "pixel_budget", + "status": 413 + }, + { + "name": "mime_spoof", + "status": 415 + }, + { + "name": "filename", + "status": 400 + }, + { + "name": "unsupported", + "status": 415 + } + ], + "small_alpha_exif": "PASS", + "private_s3": { + "anonymous_get": 403, + "outside_prefix_denied": true + }, + "stage_before_commit": { + "upload_status": 503, + "metadata": 0, + "fresh_orphan_preserved": true + }, + "commit_publish_and_restarts": { + "job_id": "51f7ee76-3566-4837-a91d-74d5a0afec7b", + "api_restart": true, + "broker_restarts": 2, + "link_queued": true + }, + "crash_after_objects": { + "attempts": 2, + "objects": 4, + "fence": 2 + }, + "commit_before_ack_and_duplicate": { + "ready_unchanged": true, + "ready_replay_rejected": true + }, + "bounded_retry_replay": { + "max_attempts": 2, + "retry_available_at": "2026-11-10T18:47:14.297456+00:00", + "first_backoff_seconds": 2.492, + "failed_attempts": 2, + "audits": 1 + }, + "permanent_stage_error": { + "attempts": 1, + "error_code": "invalid_image" + }, + "renderer_process_group_timeout": { + "deadline_seconds": 2, + "elapsed_seconds": 2.27, + "cwebp_observed": true, + "descendants_after_failure": 0 + }, + "staging_cleanup": { + "ready_stages_remaining": 0, + "linked_original_preserved": true, + "failed_retention_days": 7, + "expired_failed_stage_removed": true, + "expired_replay_rejected": true, + "unconfirmed_orphan_removed": true + } + }, + "started_at": 1794336125.445686, + "resumed_at": [ + 1794336751.851368 + ], + "finished_at": 1794336891.683626 +} diff --git a/evidence/lab5-go-tests.txt b/evidence/lab5-go-tests.txt new file mode 100644 index 00000000..b0e96060 --- /dev/null +++ b/evidence/lab5-go-tests.txt @@ -0,0 +1,14 @@ +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto (cached) +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 14.228s +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 2.227s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password (cached) +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.666s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat (cached) diff --git a/evidence/lab5-integration-initial.txt b/evidence/lab5-integration-initial.txt new file mode 100644 index 00000000..0b97ef45 --- /dev/null +++ b/evidence/lab5-integration-initial.txt @@ -0,0 +1,355 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 460.3kB +Step 1/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/27 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/27 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/27 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/27 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/27 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/27 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/27 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/27 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/27 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/27 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/27 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/27 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/27 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/27 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/27 : COPY internal ./internal + ---> 44f1a9efcc66 +Step 19/27 : COPY migrations ./migrations + ---> acefcd8a83af +Step 20/27 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Running in 6f0922eb42d3 + ---> Removed intermediate container 6f0922eb42d3 + ---> 2cda01161c62 +Step 21/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in 26952912ceb7 + ---> Removed intermediate container 26952912ceb7 + ---> 889afe1963f8 +Step 22/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in 04665a280741 + ---> Removed intermediate container 04665a280741 + ---> 43cd9030aa04 +Step 23/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Running in 5b0325120af9 + ---> Removed intermediate container 5b0325120af9 + ---> 66906c7f16a8 +Step 24/27 : FROM builder AS tests + ---> 66906c7f16a8 +Step 25/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 7e67bb289391 +Step 26/27 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in 1b5b33f07921 + ---> Removed intermediate container 1b5b33f07921 + ---> 3aba3008d340 +Step 27/27 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in bb4863e04cfd + ---> Removed intermediate container bb4863e04cfd + ---> 471b66a42df0 +Successfully built 471b66a42df0 +Successfully tagged mtusi-messenger-tests:lab5 +e5yt4nofpygdupx0vhbijo6dj +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestMessageStrictFields +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.013s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsMalformedJSON +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:44 request_id=78adaf84-7aaa-42f4-a084-a2db8cffcf98 method=POST path=/api/v1/users status=400 response_bytes=140 latency=131.98µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsMalformedJSON (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestHidesForeignChat +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=1bf6ff0c-9371-4b01-9941-77e94d023b10 method=POST path=/api/v1/users status=201 response_bytes=141 latency=365.251444ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=f50eb79b-cca7-4993-8191-8734738e9c40 method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=398.873152ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:45 request_id=5a967846-e8a8-449a-a09d-c8d365025b46 method=POST path=/api/v1/users status=201 response_bytes=147 latency=381.399429ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=714a25ea-10d1-4b85-8c24-ec4d371bb7b6 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=422.991476ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=2345f91f-0c23-45d3-a491-8dfd5556d174 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=188.281µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=0336f4ad-d624-4ecf-9cdf-51301294d0e8 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=284.797µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestHidesForeignChat (1.57s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsEmptyMessage +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:46 request_id=44c2744c-4cd0-4c7e-992d-a4dfb0b061b6 method=POST path=/api/v1/users status=201 response_bytes=157 latency=368.07845ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=af848f3f-b52b-4af3-85f8-bd965300d1ee method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=366.256457ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=23df23c7-578f-4c76-a10b-104f2426b9cd method=POST path=/api/v1/chats status=201 response_bytes=163 latency=133.647µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=b90bf297-9272-4bfb-a83f-47b326cb5ce1 method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=152.233µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsEmptyMessage (0.74s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=178aa11a-597f-493f-b227-c667f089540b method=POST path=/api/v1/users status=201 response_bytes=149 latency=351.691662ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:47 request_id=2f3ba9cf-b14c-4ae9-adfd-ec9f3028a50e method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=367.085423ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=0a727f2e-dc6b-4c93-8639-de50d77626b5 method=POST path=/api/v1/users status=409 response_bytes=170 latency=359.315715ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestRejectsDuplicateUsername (1.08s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=b3142e33-707a-46f5-b178-35509037b4cc method=POST path=/api/v1/users status=201 response_bytes=151 latency=409.355414ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=448e921b-0b3a-434c-8961-b6aa51530d88 method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=387.814711ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=8cf5775f-be80-476a-8f65-1fae6ba4d114 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=162.61µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=9e3a4eaf-e3f0-4c1c-88a7-651b43fde65c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=266.503µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=57d23cd6-2482-4656-8621-670b43ef8d3d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=186.906µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=92183f25-6ab2-4c74-a970-9cf52a20d989 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=167.945µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=8da17cdd-9817-47dd-89e5-156bf0009a30 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=167.528µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=364216f6-407d-4f49-97ac-1d79491067b4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=176.779µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=7abdd979-4157-4d85-bdb7-41b31fdb5612 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=130.438µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=30e71f25-5fcc-47fa-9349-eaa52423b428 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=134.814µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=17a01d04-e615-4d86-8016-fbf560e6f1e6 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=380.521µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=f9ebd8b6-f40a-49b9-8c46-03524ae60c7e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=183.197µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=ec02dec4-7e4c-4fdd-b05e-0115f5485111 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=174.862µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:48 request_id=90cae6ae-fe4e-4e2a-8765-2ad81c9abaf1 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=114.519µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.81s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:49 request_id=0430c49b-26f0-4036-b772-e6267ecd0b14 method=POST path=/api/v1/users status=201 response_bytes=153 latency=370.09502ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:49 request_id=7f5a51e7-7002-4917-84b0-dc9725be54a9 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=360.387497ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=a349814a-0888-445e-9726-a33a6712c43c method=POST path=/api/v1/users status=201 response_bytes=153 latency=360.66339ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=fb3e86ae-a739-4e7d-bc78-559fadbbf787 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=394.640836ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:50 request_id=0fc70956-e867-4377-96e5-329dda30cd31 method=POST path=/api/v1/users status=201 response_bytes=153 latency=367.006565ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=0f9813dc-7c0d-4163-9793-882e84468d33 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=353.933555ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=4440bfa8-9d31-4297-9343-80ded2259f6a method=POST path=/api/v1/users status=201 response_bytes=153 latency=375.799206ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:51 request_id=56c0bfa7-4938-47a5-ac31-543cddca96e9 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=366.48411ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:52 request_id=36983337-938e-4c9f-868a-7cc2f1525f5e method=POST path=/api/v1/users status=201 response_bytes=153 latency=373.712012ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:52 request_id=a2853b02-5099-4cbd-aa6c-cbfb617a289d method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=505.163831ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:53 request_id=2170619b-d50a-41a7-a77f-d38fa3fe45a2 method=POST path=/api/v1/users status=201 response_bytes=153 latency=379.44649ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:53 request_id=7fccfc33-4433-459d-929d-c59c71d7d620 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=440.462082ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=f235d12f-6bf2-4483-93e5-5515c87d8b68 method=POST path=/api/v1/users status=201 response_bytes=153 latency=467.557845ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=c039f7d0-879a-4c87-aa6f-b26a7b0be534 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=380.753972ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:54 request_id=e084cc17-3e34-4b4d-a505-8ee4ee3c2ed9 method=POST path=/api/v1/users status=201 response_bytes=153 latency=430.362118ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:55 request_id=77f5d7ed-f8a4-44bb-a7c6-35105b72861c method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=361.400539ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:55 request_id=2cee44b0-3bde-4274-a345-2a450e37d592 method=POST path=/api/v1/users status=201 response_bytes=153 latency=387.358501ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:56 request_id=8351a9bb-2d81-485e-9461-d3e1e180b1c7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=478.984823ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:56 request_id=4784bfab-a8bd-47ce-b569-4b7e519d4f38 method=POST path=/api/v1/users status=201 response_bytes=153 latency=694.465485ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0876d022-4dd6-45ff-9a5d-74b7b4dece6e method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=507.870998ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1af0e642-3a02-4cd6-88eb-295353c88b7c method=POST path=/api/v1/chats status=201 response_bytes=514 latency=303.26µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=72b9ccc9-d30f-4da3-9662-05f15722602f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=651.443µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=81eb5c5b-f9a3-4403-9b8a-5ed360f4a78d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=359.06µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=95416f6f-e84b-4532-bf7e-4c70850733ca method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=594.975µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c52d75be-211a-414e-bf1d-ce287a437a5a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=253.001µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=3acdd29e-915f-4252-9928-1bb1561a6f8a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.628µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=401c57a7-dda1-45a8-854e-1f90fec29b4d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.086266ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b1a2a6f0-d442-425a-9185-b06af53e0287 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=495.209µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e28e56e9-5b4b-4634-83a9-8ffa261246d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.567434ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1be19fc0-187a-4240-b396-51f254a28846 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.795385ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=83f80360-8ebb-40c8-b3f9-83b596a60dba method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=201.617µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f56ff57c-7170-41f5-b81f-ace5f259000d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.246253ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=51e5b495-8329-4087-afb7-be0ec16db915 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.059136ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cdc680e8-fda4-45ab-9cc2-976891d745cb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.058181ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4f378cb9-5335-4a3f-a204-c04c512bc1ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=270.587µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d15ea22e-b4d5-44ba-ad3f-a0d1cbded11d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.963708ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ff677057-1037-4998-a7ee-d3ed7516cc16 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.902029ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2edd5096-e6f7-41ef-a265-8ec8beeae9f5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.739251ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=75d1f157-564b-46f0-80f0-a88d7da83642 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.940161ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=61cdfd52-4d67-4af2-a3cc-b7384ef86be9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=191.199µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fc5faa10-3c15-4e66-9f27-4de48fefa69a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=161.194µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6dec8a6b-f059-4615-9f28-7c0e265bdced method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.403µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0ba37f3e-5012-4648-9cb9-fe6a9e391286 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=263.753µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=13417c22-fafc-481a-abd5-456111490e41 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=229.539µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=19f8ac4f-fc2a-4a64-9096-3a6cb8c0dc12 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=443.95µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f1ca84e3-d6f6-4a64-bd98-e2aa24f96793 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.696µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b6ab587d-f519-469f-88e8-9aa15221cfb1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.111µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=12ec636a-0fbf-4c26-9834-59363cc0e5d9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=346.683µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=94742954-5139-412c-940f-19dde72f8d50 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=217.537µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2e6d7919-71d0-4411-9f86-a5312c7ea148 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=237.707µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=2acd4a86-79d3-47ac-9370-490faba63dd8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=222.621µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9fd113a4-8b1f-487d-83a0-98f21d3106a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=356.06µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=51e37132-149a-4c3b-98c9-cfade19a3ecc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=299.883µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=056f9612-6681-49cb-be83-a5188a4ad784 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=198.45µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=f0933a5e-5e7d-4f67-9d21-b3cdc26b5ee7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=367.978µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fd5381a8-23d5-4e49-a2b3-a7446b8dc94a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=368.145µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=18cb05e3-b296-47b9-b011-996dd1069d13 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=680.406µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=8b636c5e-df70-4e1c-907c-8a516890d49e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.473581ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5b7fa9d4-2545-4b7a-bd14-e2a0391d895d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=219.078µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7e5258f4-6322-4d5d-857d-7724852f18a5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=179.405µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=700838c9-da94-4e08-889e-e3346b73d285 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=197.325µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cc55fc35-3e25-4d9c-b4ce-1aa9f7a296ed method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=193.741µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d21bae1c-8414-4fef-ae5e-59ecc5c8e481 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.612µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9ae11a53-4372-4cf8-9459-bb9055e5bb9f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=268.212µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4ae0bd79-fb34-41fa-aaff-47459dead558 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=177.614µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c49603b9-6709-4f40-826b-0cb39c439bb5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=170.237µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ef856d83-34ef-46ee-b047-82ceaa8c3813 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=195.325µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=948df662-0eea-4e28-ac14-15d972735bfe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=277.755µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6e05a773-85ea-4308-873b-f413fc23ab67 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=210.077µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=741af3a0-5dbf-48ad-a053-d941cd97ee49 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.989µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=81450fe0-2922-4349-9f40-d3eb9199456d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.193µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=47bffcec-6f7f-4bba-8fc5-321e6d954324 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.028µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=4fe65afb-595d-4227-b676-ff0745afdc77 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=22.151061ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a30669a7-089e-497a-b5ac-d7e3a3330b59 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=221.871µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7f746257-b6b6-464c-a964-f4b666230e3b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=278.297µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cb03cc55-07b0-4c0f-8ff6-73ca6916e076 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=252.209µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=92d14786-6c5f-4cdd-89cb-fb75ef8a2375 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=225.58µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=3c195e16-268b-494b-ad32-9135c3ea6b9e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=388.899µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e1f9a9e4-3ae3-477c-8fd6-c8d068d8f68a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=636.358µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e230085e-920a-4cda-ac31-5a08e846406d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=502.085µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b26a0e99-c1cf-4895-8a7c-b87f9010e4bf method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=962.245µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=eab81116-9035-4834-8d33-f179a4fc5757 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.862814ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=d8226a69-ff35-4e40-99dd-8c0c42cd15ba method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=358.143µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a1d94b08-0423-4a8e-985e-8a62e07c0071 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.488835ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ce158939-7a9b-4829-9e30-a9570247b067 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=309.719µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7694a9dd-aad5-4b96-9d52-bf022c2d6101 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.671325ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a75cbf2d-24f5-44ff-8b3e-2efc8858daeb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.563808ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=cccd74dd-6055-4538-b589-6439bb2e2249 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=199.409µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=682cf78c-492a-4e2e-8d0f-600709d40332 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=268.004µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b0fe5b42-55a9-4080-b152-17958d1f9f72 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.447µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=01ea3680-f345-41b6-9cf5-eaf5a42f98ce method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=201.368µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=41a3b666-77e0-45cd-903a-59c37d094a19 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=209.66µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=52fb0977-6696-48ed-914f-56bb16207ec5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=314.261µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a306642c-bb87-41a5-9c51-8d3dea67923b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.67µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=47909112-0b12-4627-a977-eabc090eec23 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=277.297µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=628fa77e-5549-4bca-8ffb-671089e4a4e4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=227.83µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=115e720e-d906-4299-a557-478100f07ffa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=561.136µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=fdffc6bf-74ff-4e02-a849-d86f8469edf9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.823µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=70c555f2-9f81-45c3-9cb1-85b4e80106f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=188.031µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=897b8deb-775f-4a0d-ac02-b8eca6e6a4ca method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.445µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5b02ee7d-ae2b-45fa-8e5d-e796648368a4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=278.255µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=59aff7ee-4d4d-487f-89dd-a9b9e1df8705 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=228.664µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6d858392-c3f1-403d-82d7-3790e81b96da method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=232.164µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=5f077336-b854-49e5-8642-22dddecd1680 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=287.299µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=44d32f91-3883-4a4f-9722-4718d1f29ed6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=363.228µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=b043468b-dcd8-4085-b21b-72427260aa7e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=265.67µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4c49e7c-e296-4263-beb6-232f11a79069 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.071µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=22a15da2-c0e5-42ca-b117-18ecea6f4be5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.155µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=9c6590f9-f07f-4bdd-a3fa-ccbfa2b34080 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=296.383µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=06413c5e-5e2f-4a5a-b3d8-72e79e095b1d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=213.16µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4fdcea5-2cb3-4852-a3e6-8f2cca4150c6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=235.748µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1e5ec364-bd31-459a-9e20-4df15f9201e5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.861µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=97042c78-9f32-4690-bed4-3fdd12e6f90c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.237µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=1294c60c-07eb-49e2-9688-8273b867a7b0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=410.319µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6b1d6ba4-31d5-4eca-a5cf-f4ab203fc38f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.275µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c9717b03-a4c6-437b-80b3-319a1b010200 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=293.008µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e8d813c8-6743-4004-a193-4f9517cc29f0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=237.081µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7163a415-e1ce-4c92-88e9-87d3e7b5ebc4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=993.375µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=e52dfded-ef1e-4815-9507-8b4c57b32403 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=368.062µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=8c03578f-5787-409f-b5ab-e61013240c28 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=330.056µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=ea8a1f84-ab01-4d6b-b4b2-1edd2745ea4a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=362.228µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=7bd598d0-dfbe-4226-9d16-8815ace71d18 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.28755ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=a36de45d-8bf9-4ec2-ae7a-ab0d5badfd91 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=970.455µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=79982f51-85fd-4e19-b8df-6c6865ed1929 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.137775ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=6045807f-6a92-440d-b54e-dc7b4eee2e76 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=879.523µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=c4eadc24-2b1e-4318-81cc-90c604831194 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=669.904µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:19:57 request_id=0eb733fc-c439-41b9-b979-c46d54fd5a5b method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=583.306µs +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestConcurrentMessagesAreUnique (8.46s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.752s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.02s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFilenameAndCancelledUpload (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestValid16BitWidePNG +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestValid16BitWidePNG (0.11s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestAttachmentTimestampsUseFixedMicroseconds +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestAttachmentTimestampsUseFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFitGeometry +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFitGeometry (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestOrientationsAndAlpha +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestOrientationsAndAlpha (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestEXIFOrientation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestEXIFOrientation (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestFenceIntegration +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestFenceIntegration (0.07s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.223s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestUnicodeAndSalt +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestUnicodeAndSalt (2.38s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.400s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestS3WireFixture +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestS3WireFixture (0.01s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:84: goose up: 2026/11/10 18:20:05 OK 001_initial.sql (21.11ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 002_idempotency.sql (1.75ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 003_attachments.sql (5.64ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 004_message_attachments.sql (1.97ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 005_async_attachments.sql (5.92ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 006_attachment_jobs.sql (4ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 007_attachment_outbox.sql (2.56ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 008_attachment_replays.sql (3.23ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 009_backfill_pipeline.sql (2.32ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 OK 010_storage_cleanup.sql (817.31µs) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:05 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:85: goose up: 2026/11/10 18:20:05 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/registration_race +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:121: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/rollback +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:139: failed second member INSERT left 0 chats and 0 members +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab5_tests.0.wqzm1lukwaby@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab5_tests.0.wqzm1lukwaby@colima | lab4_test.go:321: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:14 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:15 Actions:[{Kind:orphan Key:original/7a8e9f88-b271-4b9e-9431-b9aa433f1489 Applied:false} {Kind:missing Key:original/76f08381-064d-45b3-afa7-3e8bcb6a5dc3 Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:15 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:7 Actions:[{Kind:orphan Key:original/7a8e9f88-b271-4b9e-9431-b9aa433f1489 Applied:true} {Kind:missing Key:original/76f08381-064d-45b3-afa7-3e8bcb6a5dc3 Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:100 ListRequestsEstimate:1 StatRequests:1 DBQueries:9 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:3 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:193: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Limit (cost=0.29..17.99 rows=51 width=142) (actual time=0.013..0.032 rows=51 loops=1) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: -> Index Scan using messages_history_idx on messages m (cost=0.29..347.47 rows=1000 width=142) (actual time=0.012..0.027 rows=51 loops=1) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Index Cond: ((chat_id = 'f74d97f7-bb21-47ea-9177-5f171f6cead5'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Planning Time: 0.034 ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:199: Execution Time: 0.054 ms +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:204: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/idempotency_race +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:254: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab5_tests.0.wqzm1lukwaby@colima | storage_test.go:268: goose up: exit status 1: 2026/11/10 18:20:06 OK 001_initial.sql (2.39ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 002_idempotency.sql (1.26ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 003_attachments.sql (2.48ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 OK 004_message_attachments.sql (2.81ms) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | 2026/11/10 18:20:06 goose run: ERROR 005_async_attachments.sql: failed to run SQL migration: failed to execute SQL query "ALTER TABLE attachments\n DROP CONSTRAINT attachments_status_check,\n ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')),\n DROP CONSTRAINT attachments_check,\n ADD CONSTRAINT attachments_check CHECK ((status='failed' AND error_code IS NOT NULL) OR (status<>'failed' AND error_code IS NULL)),\n ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',\n ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),\n ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,\n ADD COLUMN staging_key text;": ERROR: column "variants" of relation "attachments" already exists (SQLSTATE 42701) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- FAIL: TestIntegration (0.64s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/registration_race (0.03s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.07s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/keyset_and_explain (0.09s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestIntegration/idempotency_race (0.02s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- FAIL: TestIntegration/goose_legacy_and_down (0.05s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 0.667s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab5_tests.0.wqzm1lukwaby@colima | === RUN TestPasswordContract +mtusi-lab5_tests.0.wqzm1lukwaby@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab5_tests.0.wqzm1lukwaby@colima | PASS +mtusi-lab5_tests.0.wqzm1lukwaby@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.013s +mtusi-lab5_tests.0.wqzm1lukwaby@colima | FAIL +mtusi-lab5_tests +FAIL: tests failed +make: *** [test-integration] Error 1 diff --git a/evidence/lab5-integration.txt b/evidence/lab5-integration.txt new file mode 100644 index 00000000..0069f7b7 --- /dev/null +++ b/evidence/lab5-integration.txt @@ -0,0 +1,389 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 461.3kB +Step 1/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/27 : RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 15f4b2e3468a +Step 3/27 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/27 : COPY migrations /migrations + ---> Using cache + ---> 01a2e64cbb7d +Step 6/27 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 7382e24476ff +Step 7/27 : USER 10001:10001 + ---> Using cache + ---> 6c54ddff6a36 +Step 8/27 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> 7e8134892cf2 +Step 9/27 : CMD ["up"] + ---> Using cache + ---> adb6e8d9b58a +Step 10/27 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/27 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/27 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/27 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/27 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/27 : COPY cmd ./cmd + ---> Using cache + ---> 7d74a0366782 +Step 16/27 : COPY contracts ./contracts + ---> Using cache + ---> 89e9eb49cf2e +Step 17/27 : COPY dto ./dto + ---> Using cache + ---> b3608ba9d910 +Step 18/27 : COPY internal ./internal + ---> 420381f245d1 +Step 19/27 : COPY migrations ./migrations + ---> 244708889b45 +Step 20/27 : ENV GOFLAGS=-p=1 GOMAXPROCS=2 + ---> Running in bf624bf36a2e + ---> Removed intermediate container bf624bf36a2e + ---> a86cf22e4c7d +Step 21/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in f0e750b5bd34 + ---> Removed intermediate container f0e750b5bd34 + ---> e1ea9cbf840a +Step 22/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in d70ebca4dc47 + ---> Removed intermediate container d70ebca4dc47 + ---> 4669d728553f +Step 23/27 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + ---> Running in e5f260b0016c + ---> Removed intermediate container e5f260b0016c + ---> fe51ba375215 +Step 24/27 : FROM builder AS tests + ---> fe51ba375215 +Step 25/27 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 3ef73620b0a6 +Step 26/27 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in 4b59132a45e1 + ---> Removed intermediate container 4b59132a45e1 + ---> 43905e46829a +Step 27/27 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in 9d6cbda904bc + ---> Removed intermediate container 9d6cbda904bc + ---> 037b4f43fd3c +Successfully built 037b4f43fd3c +Successfully tagged mtusi-messenger-tests:lab5 +4sbaveec2mbbm7q290z3mc4bh +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/worker [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestMessageStrictFields +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.016s +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsMalformedJSON +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:34 request_id=a120ebf4-66d8-4d9e-b9ef-404d1f0b8ad1 method=POST path=/api/v1/users status=400 response_bytes=140 latency=186.094µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsMalformedJSON (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestHidesForeignChat +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:35 request_id=525ea333-7fbc-4427-8ae2-082330c0b56f method=POST path=/api/v1/users status=201 response_bytes=141 latency=429.523996ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:35 request_id=3a55c465-0b19-4fe6-884c-12b97845202c method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=400.777506ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=91bc227a-7b92-459b-a9c7-c4a640a1ca3c method=POST path=/api/v1/users status=201 response_bytes=147 latency=384.621416ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=da2274f8-4ab2-4c54-981b-4b6375c89c48 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=418.414288ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=d5be3cdc-dab8-48ae-8e97-027a7839b8d0 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=196.223µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=c68e4735-48e5-4446-acbc-b756b3f00c63 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=55.016µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestHidesForeignChat (1.64s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsEmptyMessage +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:36 request_id=f144edb2-0a71-4953-9712-670b4307d049 method=POST path=/api/v1/users status=201 response_bytes=157 latency=406.815108ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=a7cab490-48fc-49d4-8104-8eb04cb0e41f method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=405.008764ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=41a94031-c08c-4c45-9b9b-8ad54a797f9b method=POST path=/api/v1/chats status=201 response_bytes=163 latency=106.863µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=f2a1b6f2-94d2-4f39-9bac-ebae19dab481 method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=151.21µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsEmptyMessage (0.82s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:37 request_id=a4e26cce-1333-49cf-ac98-37aba8e40cf9 method=POST path=/api/v1/users status=201 response_bytes=149 latency=384.550494ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=bf6b6a52-a044-4104-b8df-71842f061941 method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=370.705527ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=53201beb-d49c-4fc0-8f28-815f7579b8f5 method=POST path=/api/v1/users status=409 response_bytes=170 latency=370.869332ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestRejectsDuplicateUsername (1.13s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:38 request_id=ac404967-cf50-4886-ab88-75fba0d66fbe method=POST path=/api/v1/users status=201 response_bytes=151 latency=399.448651ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=f4f96a97-08ed-49bb-8074-506429860102 method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=471.634839ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=eadaf592-9287-4360-86c8-4387b3829097 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=126.203µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=2ac4a3fa-0fb8-40f0-b3a1-7af36163d37a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=197.723µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=52535278-cbce-435e-bd1e-37a433e56dfa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=179.885µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=5b11d2a6-386b-47be-aff7-317eb4fa2707 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=160.38µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=861d72de-e7b9-4bbc-a5e5-b366f4387472 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=140.582µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=0bddc2b2-b304-498d-9e09-1470cc7d31a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=119.618µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=2dc5031e-2f06-46e2-8ac7-0761ddad5e4d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=153.17µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=0d0e5d73-d66f-4083-a5d0-686c6dec4c3c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=159.254µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=ac89c25b-9da9-42e3-9cd1-0c1e1f264ee1 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=390.862µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=de3853f3-74ae-4083-bcc8-598bb45ab9b9 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=242.07µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=43b8dc81-f545-4a36-a052-6dea583fdff4 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=175.425µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=305b99d8-fcb8-4fed-a648-24c5f4e817dc method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=121.993µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.88s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:39 request_id=f62e7a14-b7ae-4c24-bc64-cc47ac50dc93 method=POST path=/api/v1/users status=201 response_bytes=153 latency=376.003337ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=9922e1a1-b272-4cb4-807e-9c872404a471 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=371.739811ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=413c97b5-8149-4c0d-92d5-4d9120882e9a method=POST path=/api/v1/users status=201 response_bytes=153 latency=371.796731ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:40 request_id=417099fe-cbba-430c-ba5f-694776a15347 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=401.933506ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=2cd09de2-85fe-4ad0-bc87-8187a366a970 method=POST path=/api/v1/users status=201 response_bytes=153 latency=395.859968ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=409307c0-72d4-44c4-9aa5-33b3c5752aa8 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=396.505304ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:41 request_id=ebec8d8c-e6a1-4c8c-8122-e0130516d2fc method=POST path=/api/v1/users status=201 response_bytes=153 latency=373.820076ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:42 request_id=385f8376-dbcc-4c0f-bd5b-22e6cbf989a4 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=408.30886ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:42 request_id=2e8f957b-af2d-4f82-890c-17ee5ddd9bdc method=POST path=/api/v1/users status=201 response_bytes=153 latency=401.205396ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=41a56fb0-e2cb-4312-a57c-6457c00f79fe method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=381.081401ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=121e271b-7634-48e5-8395-90b992a18b75 method=POST path=/api/v1/users status=201 response_bytes=153 latency=400.063202ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:43 request_id=cf890726-06a9-4a26-b148-48ad9d16c1f4 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=377.397779ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:44 request_id=1d7ed2cb-025a-487e-9995-06a61e6ea59f method=POST path=/api/v1/users status=201 response_bytes=153 latency=380.557862ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:44 request_id=fb4f8e2f-725e-4559-beb8-d828632a0f1e method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=395.192982ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=357d86d1-d0be-479a-83df-b91f0da90c9f method=POST path=/api/v1/users status=201 response_bytes=153 latency=371.774229ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=38b565f2-da72-489b-8149-9e55aa927f0d method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=442.382653ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:45 request_id=b7b29434-a765-45ab-a3df-bcd2b31e5642 method=POST path=/api/v1/users status=201 response_bytes=153 latency=372.279093ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=731b7f36-a09c-46b0-beb5-69cec9ba6abe method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=370.335795ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=42c59da2-0dcb-4be0-a7f4-0de2f6a388a6 method=POST path=/api/v1/users status=201 response_bytes=153 latency=375.164369ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b59fa1d6-c07f-4b0c-a80f-df0c91db11da method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=405.93703ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=092cd05e-deb7-4378-b811-25c9f7cd0bad method=POST path=/api/v1/chats status=201 response_bytes=514 latency=186.533µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8ca6b6d5-4ae2-4464-8b7c-abdb8835cadd method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.303µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=9904aa19-424a-4c52-bcba-ec103dae6fab method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=158.803µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=439132c6-6030-45ae-bb6b-032782ad78d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.073µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=62bee418-056f-4d68-ad0b-d1494dae9149 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.233µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6f1c1df5-2e1a-4a1f-a1e2-07c9b356c41b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=168.571µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e7097d00-0825-45a0-9703-43848b1b86f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.857µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=94a4778a-b603-4947-99e6-7262cb63a02c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=178.462µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=61f7f04e-7749-4007-80fa-5348d23cc79b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=158.596µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=339f3ebd-f56b-46c4-afb4-d838b6d19e22 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.095µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=664575f1-de2b-4035-8290-02621be7def0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=567.213µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c6933c53-6a84-4462-bd1b-a27c6104dce5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=276.674µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0d005f58-b73e-4c0c-80ef-90333f841d6d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.219435ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=571fead6-abcc-4205-a7fd-7d48ce65449a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=875.838µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=ebb9eb40-526f-4139-a951-eb89822d28d7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.808419ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8328795d-6c01-4702-831b-9679f1c66839 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.215µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b3030fcc-c444-49d7-a1c2-6132b1baa8da method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.662µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2c933d51-f44b-4fac-a675-72c85774c512 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=484.438µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c41778c0-56b2-49c1-9a82-1159463cffc6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.414535ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=38ff6efb-454d-4772-8ba8-20683532d351 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.087575ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4610cbff-d3c7-4083-a7b1-01621d4129cb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.651312ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=8c2687e3-8b73-4720-bfa5-7596c6fdd5ed method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.211241ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=86372bfa-614a-4f92-bf1e-df5b302505bc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=3.222457ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a7cd6676-8436-4544-966b-bbc0c0241148 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=766.079µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=70bdf943-11df-447a-bfe1-3a5c22172a0d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.299µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=41665126-c28d-4623-943b-12c375e7cd4e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.898µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f53ba09b-53a2-4071-b898-f828b948e171 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.423µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=673d03d2-6c91-435f-8b8c-8aff5b58c314 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.455µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fea6b8ec-d431-492d-b61b-9403f165f818 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.863µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=74604d89-3eac-4ee0-8ff1-a70255fe6908 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.908µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=42cf5708-e2e9-45c7-addc-375633b507eb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=157.106µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d7d86eee-ce6b-4c36-8b29-559b33612678 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.241µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=694a46d1-d858-4046-96b4-832eaa0f9983 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.09µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c8afd69d-af62-4dd5-a275-b9446299c738 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.378µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4cf77251-b87b-445b-b1b4-14096daa4bc4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.543µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2fb59757-2e18-4d3d-855a-2e258a19414f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.856µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f1d91a2c-c587-49ea-9868-7c70f1abffeb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=148.415µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1d2a1995-c266-4a4b-9544-b54480cebd2c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=139.31µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=3912cb0a-9060-4302-84a5-5d7c786d71c4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=137.861µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=93908403-558e-450b-866a-58f5a321c88d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=286.317µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=5ee59046-acf0-48c8-bb7d-5f21abe094d8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=198.163µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=23c591a6-d56b-4018-b130-861d8e0589c8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.999µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=31bbf0d9-a4af-4848-aefa-a771dcd55c40 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=309.453µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1198591d-c43b-4c9b-bc76-4fad9f4bc40c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.153µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=17e0a733-80ae-4f83-ae88-2fab97a5f044 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=156.734µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=7f40a24a-d447-45d6-beae-8b920e9bdbc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.592µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=bb29dbe8-0483-400a-9b60-02cd83c15485 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=505.546µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e0da57fa-9e99-4f0c-8516-abdd4af85e46 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=725.602µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=428d1db9-9475-44e5-9eb6-c2c0b92bf418 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=491.226µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=eea31235-f85e-43df-bc23-9f6f0e7e7adb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.944666ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=57a62d96-2f93-408e-9eb2-ce3e13c70a29 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.149243ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=1f5dd0c0-80f3-443d-a7ae-62c5aed88439 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.051817ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2de77a11-98c7-45f5-91da-31b23e2d91c5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=468.836µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=ac732004-5639-4c64-a815-de0e49ed3321 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.150691ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=54228bc8-0df7-4a23-b2d1-03ae892bd7f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.314626ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d0d42de0-7fb6-46ae-b3bf-ca0d5950b03f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.556204ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=68605d50-22f1-4dcb-88b3-a956b64e5cb3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=366.981µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=d1b73e5c-4917-4a78-9c82-8add850b5079 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=282.924µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=dddebfc6-6f64-479f-80c3-87030922367c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=295.381µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4f3b527c-4137-4cb2-9473-49d56a420bbb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.049µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b9b06cbc-f7ac-44d6-9811-0fa2543cca70 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.34µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=e732f927-fe6d-40b9-a9ad-7782274e3650 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=345.377µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0f10e634-118c-4cef-990c-1a7abb919ab5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=360.07µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4be5aad6-3a1d-494e-b0de-29e3a2972552 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=183.76µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=eddafdc2-c9b9-43d1-88f4-10f1c56071d2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.964µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b6c0615f-dce5-4631-aa5f-7dc2bf061681 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=290.705µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a6c82fb3-0419-4bfb-badb-e4f45ca50d92 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=347.984µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=41704ab7-1120-4aba-84e8-628dc70275b1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=589.562µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=76201778-5847-487a-b229-629354621449 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.083023ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c0d50461-50d5-45a9-aaad-5af34d30a440 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=120.52µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c1d344b9-dba4-4392-9e54-1efa06170360 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.605µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=f64ae733-f80b-4c1c-a953-e9ff5d10113b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=128.011µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6e66ae23-1cdf-4620-a8eb-0c12e6f01495 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=128.425µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fb60631d-2a25-4a80-bddf-6667c0e12e2c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.111µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=17c0b29d-7dd4-4b61-ade5-695194f19a68 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.824µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6aa57338-428c-45a8-9d77-bd8c524f58d0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.685µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=c886be6f-ee67-44a8-abe6-84e489cf975c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=196.921µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=de16609f-4c14-412f-a4f1-f28d527599db method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.891µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=6876ce9a-90c3-4431-a056-7508ae43b034 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=118.988µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=2849e735-0d5e-4ab1-9932-d8be2bcdf65c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.603µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a883d328-3055-45f2-b971-ce43be573940 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=125.445µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=9f5f45ed-6b88-458d-9525-9326030919e1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.356µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=0f74d6b0-8931-464a-aa5a-f26ad12daec4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=119.94µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=fbdc3ebd-e88b-4864-8131-8ad4b89eb410 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.158µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a7ba86ee-181b-4476-a7ab-a734c4ee266c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.84µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=5e51912a-012a-4c56-9313-ceb9d44c4cc5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=261.485µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=a3130f83-d27c-4ec2-87d3-59027067ea6a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.245µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=b9e5b9b3-bbd6-4568-8753-d4351dc96839 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=147.545µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=4565e89e-caa9-4e4d-a078-a7ba9cbc479d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=140.593µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=3df97f91-bacf-4194-a2df-e91b8a94d2a8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=176.723µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:46 request_id=27f66f60-b97d-4d3e-9503-1af72c291662 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=127.225µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=64e38cb8-6eb5-4a3a-a01d-090dccca1035 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=152.636µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=ae0c3b4f-fa7e-4f02-abf9-c4e5ca5f4236 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=199.983µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=58d8552c-cda1-47d2-8881-d1a43b03ad3e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.335µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=1dab28fb-6c71-4aa4-b8d5-da8e86da9e8a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=172.743µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=ce0e5337-dc4f-4b11-8afc-1edf7653bf7f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=347.195µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=28900f9b-33f1-4266-8950-5b7ed26d3867 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=143.56µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=cfcd75a4-9fc8-4bd7-82ff-929b69ce829a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=131.59µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=9fd19029-673f-4935-9668-84a2d0e683d5 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=124.839µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=4ef3f1ae-50c1-413e-8c4e-1dafa5fb2d1c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=137.803µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=cbcb3daa-1979-4392-bc47-6f5b20ea3070 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=289.151µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=a5ea6313-9465-4d20-b068-2553d3bee370 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=776.079µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=48187e18-3ecd-4fbd-a062-c6de8a354daf method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=1.41448ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=1fc7fb98-1c6d-4376-832c-530e21630c58 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=694.317µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=abfeb221-d940-4411-a679-64678c036f6e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=606.963µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=b0e89015-1603-402a-939c-eb16b757c1a6 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=536.633µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:47 request_id=01fed6c0-16a3-4df6-9b1b-bb20b74d7fe2 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=481.751µs +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestConcurrentMessagesAreUnique (7.82s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.393s +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.03s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFilenameAndCancelledUpload (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestValid16BitWidePNG +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestValid16BitWidePNG (0.10s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestAttachmentTimestampsUseFixedMicroseconds +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestAttachmentTimestampsUseFixedMicroseconds (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFitGeometry +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFitGeometry (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestOrientationsAndAlpha +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestOrientationsAndAlpha (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestEXIFOrientation +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestEXIFOrientation (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestFenceIntegration +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestFenceIntegration (0.07s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.244s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestUnicodeAndSalt +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestUnicodeAndSalt (2.70s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.754s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestS3WireFixture +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestS3WireFixture (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:85: goose up: 2026/11/10 18:24:55 OK 001_initial.sql (16.88ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 002_idempotency.sql (1.65ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 003_attachments.sql (3.72ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 004_message_attachments.sql (1.35ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 005_async_attachments.sql (4.89ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 006_attachment_jobs.sql (2.81ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 007_attachment_outbox.sql (2.14ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 008_attachment_replays.sql (3.36ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 009_backfill_pipeline.sql (1.54ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 OK 010_storage_cleanup.sql (431.27µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:55 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:86: goose up: 2026/11/10 18:24:55 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/registration_race +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:122: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/rollback +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:140: failed second member INSERT left 0 chats and 0 members +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab5_tests.0.ei8mkr461zio@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab5_tests.0.ei8mkr461zio@colima | lab4_test.go:321: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:14 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:9 Actions:[{Kind:orphan Key:original/93a9cd5a-2728-4884-9071-a77f1e762d68 Applied:false} {Kind:missing Key:original/fc633700-4e23-4d86-a40c-6242cfad8d27 Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:5 MetadataScanned:2 BytesScanned:106 ListRequestsEstimate:1 StatRequests:3 DBQueries:15 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:7 Actions:[{Kind:orphan Key:original/93a9cd5a-2728-4884-9071-a77f1e762d68 Applied:true} {Kind:missing Key:original/fc633700-4e23-4d86-a40c-6242cfad8d27 Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:100 ListRequestsEstimate:1 StatRequests:1 DBQueries:9 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:2 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:194: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Limit (cost=0.29..17.99 rows=51 width=142) (actual time=0.023..0.041 rows=51 loops=1) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Buffers: shared hit=10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: -> Index Scan using messages_history_idx on messages m (cost=0.29..347.47 rows=1000 width=142) (actual time=0.022..0.036 rows=51 loops=1) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Index Cond: ((chat_id = '4f6f6bd1-9d3d-4b0a-b567-b15a8a04c64f'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Buffers: shared hit=10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Planning Time: 0.027 ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:200: Execution Time: 0.068 ms +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:205: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/idempotency_race +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:255: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:265: goose down-to: 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (2.55ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (467.83µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (3.28ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (3.2ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (3.22ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (2.97ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: no migrations to run. current version: 4 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:271: goose up: 2026/11/10 18:24:56 OK 001_initial.sql (1.76ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (1.26ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (478.98µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (2.98ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (6.7ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (2.98ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (1.94ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.86ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (2.12ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (485.17µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:272: goose up: 2026/11/10 18:24:56 goose: no migrations to run. current version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:279: adopted legacy schema: all existing messages are unchanged +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:280: goose reset: 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (2.13ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (705.12µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.89ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (2.62ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (3.52ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (3.49ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (4.03ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (3.47ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (961.4µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 001_initial.sql (8.51ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:285: goose up: 2026/11/10 18:24:56 OK 001_initial.sql (10.59ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 002_idempotency.sql (1.5ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 003_attachments.sql (3.42ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 004_message_attachments.sql (1.18ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 005_async_attachments.sql (3.95ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 006_attachment_jobs.sql (2.66ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 007_attachment_outbox.sql (1.8ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 008_attachment_replays.sql (2.38ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 009_backfill_pipeline.sql (1.43ms) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 OK 010_storage_cleanup.sql (484.05µs) +mtusi-lab5_tests.0.ei8mkr461zio@colima | 2026/11/10 18:24:56 goose: successfully migrated database to version: 10 +mtusi-lab5_tests.0.ei8mkr461zio@colima | storage_test.go:289: down and up succeeded in isolated schema +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration (0.72s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/registration_race (0.03s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.06s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/keyset_and_explain (0.08s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/idempotency_race (0.01s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestIntegration/goose_legacy_and_down (0.20s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.752s +mtusi-lab5_tests.0.ei8mkr461zio@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab5_tests.0.ei8mkr461zio@colima | === RUN TestPasswordContract +mtusi-lab5_tests.0.ei8mkr461zio@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab5_tests.0.ei8mkr461zio@colima | PASS +mtusi-lab5_tests.0.ei8mkr461zio@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.018s +mtusi-lab5_tests diff --git a/evidence/lab5-persistence.txt b/evidence/lab5-persistence.txt new file mode 100644 index 00000000..c7cc5f34 --- /dev/null +++ b/evidence/lab5-persistence.txt @@ -0,0 +1,3 @@ +.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-research.csv b/evidence/lab5-research.csv new file mode 100644 index 00000000..c1c4fb8a --- /dev/null +++ b/evidence/lab5-research.csv @@ -0,0 +1,7 @@ +width,height,concurrency,prefetch,offered,accepted,completed,throughput_rps,ready_p95_ms,after_accept_p95_ms,upload_p95_ms,queue_lag_peak_seconds,rss_peak_bytes +512,512,1,1,20,20,20,3.316,4761.802,4554.64,328.115,4.078925,33816576 +512,512,2,4,20,20,20,4.013,4327.625,4169.426,173.305,3.172338,56823808 +512,512,4,8,20,20,20,3.903,4453.48,4281.149,191.95,2.665913,94711808 +1536,1536,1,1,20,20,20,0.774,22792.626,21836.999,1073.912,21.476154,77754368 +1536,1536,2,4,20,20,20,0.851,21471.061,20489.723,1147.664,18.202527,140812288 +1536,1536,4,8,20,20,20,0.847,20744.497,19885.435,1134.032,16.602282,235130880 diff --git a/evidence/lab5-research.json b/evidence/lab5-research.json new file mode 100644 index 00000000..61be85ed --- /dev/null +++ b/evidence/lab5-research.json @@ -0,0 +1,2099 @@ +{ + "track": 1, + "started_at": "2026-11-10T19:03:13.613546+00:00", + "stack": "mtusi-lab5", + "worker_image": "mtusi-messenger:lab5", + "worker_image_id": "sha256:66e3c9bc004e8a21f70f7f537d56d64b847ea099719a3b756cd8da9e3bfad5f7", + "environment": { + "docker_server": "29.5.2", + "docker_cpus": 2, + "docker_memory_bytes": 2053644288, + "architecture": "arm64", + "python": "3.9.6" + }, + "worker_limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "initial_settings": { + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "4" + }, + "api": { + "MAX_OUTSTANDING_JOBS": null + } + }, + "requests_per_case": 20, + "upload_concurrency": 8, + "sampling_interval_min_seconds": 0.25, + "method": { + "throughput": "completed / burst makespan including client upload and polling", + "ready_latency": "client POST start to first observed ready; nearest-rank p95", + "queue_lag": "age of oldest queued PostgreSQL job in this chat; excludes processing", + "rss": "sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly", + "limitations": "single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA" + }, + "cases": [ + { + "concurrency": 1, + "prefetch": 1, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 6.031, + "throughput_rps": 3.316, + "ready_p50_ms": 3005.016, + "ready_p95_ms": 4761.802, + "after_accept_p95_ms": 4554.64, + "upload_p95_ms": 328.115, + "queue_lag_peak_seconds": 4.078925, + "outstanding_peak": 19, + "rss_baseline_bytes": 15818752, + "rss_peak_bytes": 33816576, + "samples": [ + { + "seconds": 0.583, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 0.01802, + "rss_bytes": 29978624 + }, + { + "seconds": 1.136, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 0.573877, + "rss_bytes": 33816576 + }, + { + "seconds": 1.71, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 1.10088, + "rss_bytes": 30461952 + }, + { + "seconds": 2.245, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 1.551341, + "rss_bytes": 30990336 + }, + { + "seconds": 2.733, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 1.994326, + "rss_bytes": 31334400 + }, + { + "seconds": 3.207, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 2.390667, + "rss_bytes": 31141888 + }, + { + "seconds": 3.691, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 2.818544, + "rss_bytes": 30248960 + }, + { + "seconds": 4.178, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 3.272491, + "rss_bytes": 31404032 + }, + { + "seconds": 4.708, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 3.762719, + "rss_bytes": 30318592 + }, + { + "seconds": 5.157, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 4.078925, + "rss_bytes": 28913664 + }, + { + "seconds": 5.598, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 15974400 + }, + { + "seconds": 6.03, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 15974400 + } + ], + "observations": [ + { + "attachment_id": "646b2385-3ab6-445d-8b78-d1a8814c4e80", + "ready_ms": 662.87, + "upload_ms": 189.316 + }, + { + "attachment_id": "dd9a1cba-9a24-4de8-be88-5ab566c354f4", + "ready_ms": 1202.492, + "upload_ms": 229.308 + }, + { + "attachment_id": "f118b697-73b8-4c02-b5cb-a32c2c26c4c7", + "ready_ms": 1211.582, + "upload_ms": 252.032 + }, + { + "attachment_id": "0b3b8787-4622-4244-9f62-2f017ac7b36f", + "ready_ms": 1782.331, + "upload_ms": 269.074 + }, + { + "attachment_id": "5158f4c7-9a48-439f-a55d-d102a6b62b72", + "ready_ms": 1782.638, + "upload_ms": 288.937 + }, + { + "attachment_id": "9c04af07-d559-47a6-8141-07fa3600f1ae", + "ready_ms": 2283.593, + "upload_ms": 301.658 + }, + { + "attachment_id": "1c5eb5a6-4c88-46ae-bbe2-5775aa37a0dd", + "ready_ms": 2291.712, + "upload_ms": 328.115 + }, + { + "attachment_id": "75f0c388-755d-42f5-963d-cf4a5d2e3a3e", + "ready_ms": 2742.883, + "upload_ms": 341.367 + }, + { + "attachment_id": "4e5b2b2f-a535-4d4d-ad66-b044d3591611", + "ready_ms": 2602.965, + "upload_ms": 218.028 + }, + { + "attachment_id": "2906e543-876e-4573-9589-b34ccd299a16", + "ready_ms": 3005.016, + "upload_ms": 232.298 + }, + { + "attachment_id": "9d93b13c-aad0-488b-964b-69eee71b6c7e", + "ready_ms": 3043.318, + "upload_ms": 205.818 + }, + { + "attachment_id": "b2b3b8fe-087b-48fe-8e1a-ac63a2af9995", + "ready_ms": 3427.563, + "upload_ms": 255.924 + }, + { + "attachment_id": "ebed9087-a7e7-4747-a007-bde8811e6835", + "ready_ms": 3459.608, + "upload_ms": 254.825 + }, + { + "attachment_id": "bf03b424-088c-4524-b987-cd3412d8f7a2", + "ready_ms": 3862.413, + "upload_ms": 233.23 + }, + { + "attachment_id": "4ea92ffb-c786-4cf6-aea9-928187c709d7", + "ready_ms": 3903.646, + "upload_ms": 258.777 + }, + { + "attachment_id": "c63b7647-09ad-4c51-9288-d96b96053ea8", + "ready_ms": 4367.295, + "upload_ms": 230.246 + }, + { + "attachment_id": "72468d5c-21c3-474e-9c9c-975580cdb287", + "ready_ms": 4347.311, + "upload_ms": 215.974 + }, + { + "attachment_id": "697018a9-f526-4d61-8d76-4ad2cd59ad70", + "ready_ms": 4761.802, + "upload_ms": 207.163 + }, + { + "attachment_id": "2bb19513-f2c9-4b58-a06d-f338afb9598c", + "ready_ms": 4715.44, + "upload_ms": 167.403 + }, + { + "attachment_id": "b9005407-6f57-4553-8521-1acc7b0fbafe", + "ready_ms": 5106.537, + "upload_ms": 142.814 + } + ] + }, + { + "concurrency": 2, + "prefetch": 4, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 4.983, + "throughput_rps": 4.013, + "ready_p50_ms": 2356.701, + "ready_p95_ms": 4327.625, + "after_accept_p95_ms": 4169.426, + "upload_p95_ms": 173.305, + "queue_lag_peak_seconds": 3.172338, + "outstanding_peak": 18, + "rss_baseline_bytes": 15888384, + "rss_peak_bytes": 56823808, + "samples": [ + { + "seconds": 0.485, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 0.069336, + "rss_bytes": 44666880 + }, + { + "seconds": 1.083, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 0.611984, + "rss_bytes": 46854144 + }, + { + "seconds": 1.67, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 1.192728, + "rss_bytes": 56823808 + }, + { + "seconds": 2.252, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 1.705331, + "rss_bytes": 46981120 + }, + { + "seconds": 2.843, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 2.197292, + "rss_bytes": 56717312 + }, + { + "seconds": 3.499, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 2.749598, + "rss_bytes": 44724224 + }, + { + "seconds": 4.037, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 3.172338, + "rss_bytes": 49577984 + }, + { + "seconds": 4.537, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 38957056 + }, + { + "seconds": 4.983, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16973824 + } + ], + "observations": [ + { + "attachment_id": "ce826562-f2d7-4646-a8cc-666fdc0e3445", + "ready_ms": 648.575, + "upload_ms": 93.272 + }, + { + "attachment_id": "14901dea-5a21-4972-b727-483944df4289", + "ready_ms": 1200.2, + "upload_ms": 75.948 + }, + { + "attachment_id": "f47aaab9-212f-4e51-96b4-a5f6560f8705", + "ready_ms": 1203.505, + "upload_ms": 110.701 + }, + { + "attachment_id": "099ea578-7850-4f8d-8405-5ded1065fda3", + "ready_ms": 1193.826, + "upload_ms": 122.626 + }, + { + "attachment_id": "408f9f25-d253-457b-ab48-c721d7009ef2", + "ready_ms": 1770.196, + "upload_ms": 158.926 + }, + { + "attachment_id": "e20381b7-ba3a-4c55-9101-0f04ac9d1146", + "ready_ms": 1807.439, + "upload_ms": 156.373 + }, + { + "attachment_id": "ef819d0c-67b9-4807-8aad-135d313de3a2", + "ready_ms": 2323.718, + "upload_ms": 172.673 + }, + { + "attachment_id": "1e74c982-9b68-4daf-bc86-b245cf8b3f55", + "ready_ms": 2243.281, + "upload_ms": 173.305 + }, + { + "attachment_id": "d1ce4c95-59b1-4f54-92f8-be6d4492b320", + "ready_ms": 2302.181, + "upload_ms": 166.344 + }, + { + "attachment_id": "4d512530-df1e-4cb7-854a-acd1b637eeae", + "ready_ms": 2356.701, + "upload_ms": 183.129 + }, + { + "attachment_id": "ab8f6f91-fd57-4358-91e7-4bd066c8cb63", + "ready_ms": 2899.235, + "upload_ms": 168.626 + }, + { + "attachment_id": "b775bd82-d85b-43b2-99e8-5c83564b3372", + "ready_ms": 2871.317, + "upload_ms": 170.58 + }, + { + "attachment_id": "49e64a58-dd48-4f80-a751-945ba319225a", + "ready_ms": 3432.304, + "upload_ms": 165.885 + }, + { + "attachment_id": "04a04ce5-153c-46be-9a61-db23f60cc080", + "ready_ms": 3456.342, + "upload_ms": 168.861 + }, + { + "attachment_id": "138ec4c7-c9ca-4588-aafb-156b0d09668c", + "ready_ms": 3868.115, + "upload_ms": 159.786 + }, + { + "attachment_id": "5bd994da-1dd8-4203-b31a-11038114d25c", + "ready_ms": 3935.046, + "upload_ms": 162.486 + }, + { + "attachment_id": "304e9528-c937-49c4-b218-2503fa608921", + "ready_ms": 3923.622, + "upload_ms": 160.39 + }, + { + "attachment_id": "ea458347-e9ac-4e16-bdc1-8bd337b7a51f", + "ready_ms": 3903.069, + "upload_ms": 153.094 + }, + { + "attachment_id": "9ba18ea9-13c3-433d-a446-a16b0e13dd19", + "ready_ms": 4344.972, + "upload_ms": 161.284 + }, + { + "attachment_id": "e96a7be8-9b5c-4711-9c52-45da94063afe", + "ready_ms": 4327.625, + "upload_ms": 158.199 + } + ] + }, + { + "concurrency": 4, + "prefetch": 8, + "width": 512, + "height": 512, + "size_bytes": 787252, + "input_sha256": "205c1114728681c54a23968d77501d0948b5abea6dfa6522bee2f91eeac3dfe2", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 5.124, + "throughput_rps": 3.903, + "ready_p50_ms": 2900.885, + "ready_p95_ms": 4453.48, + "after_accept_p95_ms": 4281.149, + "upload_p95_ms": 191.95, + "queue_lag_peak_seconds": 2.665913, + "outstanding_peak": 20, + "rss_baseline_bytes": 15736832, + "rss_peak_bytes": 94711808, + "samples": [ + { + "seconds": 0.556, + "outstanding": 5, + "queued": 5, + "processing": 0, + "queue_lag_seconds": 0.095001, + "rss_bytes": 65802240 + }, + { + "seconds": 1.102, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 0.658051, + "rss_bytes": 94711808 + }, + { + "seconds": 1.807, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 1.258819, + "rss_bytes": 82518016 + }, + { + "seconds": 2.391, + "outstanding": 13, + "queued": 9, + "processing": 4, + "queue_lag_seconds": 1.72038, + "rss_bytes": 73089024 + }, + { + "seconds": 2.982, + "outstanding": 11, + "queued": 7, + "processing": 4, + "queue_lag_seconds": 2.266387, + "rss_bytes": 66252800 + }, + { + "seconds": 3.593, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 2.665913, + "rss_bytes": 78663680 + }, + { + "seconds": 4.172, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 59797504 + }, + { + "seconds": 4.669, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 16719872 + }, + { + "seconds": 5.123, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16781312 + } + ], + "observations": [ + { + "attachment_id": "6fb4bcdb-d3a6-422e-aadf-5114682fbf6c", + "ready_ms": 1168.149, + "upload_ms": 118.86 + }, + { + "attachment_id": "38bd01ea-2541-4e0a-9273-a9056a7546e4", + "ready_ms": 1214.942, + "upload_ms": 114.237 + }, + { + "attachment_id": "9f617f89-4585-47be-a0e5-f55ce010b21b", + "ready_ms": 1243.659, + "upload_ms": 78.523 + }, + { + "attachment_id": "3248421b-96f1-423b-8c6e-ab29cbab2829", + "ready_ms": 1263.479, + "upload_ms": 102.431 + }, + { + "attachment_id": "ae7fb249-3399-4e9f-8d51-afc46c98e7f7", + "ready_ms": 2434.688, + "upload_ms": 155.371 + }, + { + "attachment_id": "84e866c0-c4a1-4996-898b-e19f90bf28f9", + "ready_ms": 2449.042, + "upload_ms": 163.852 + }, + { + "attachment_id": "0a3ebb63-4cd5-44d7-aa17-edf8972ea240", + "ready_ms": 2386.365, + "upload_ms": 166.507 + }, + { + "attachment_id": "655decd7-e199-45b0-9084-9242cba0fb63", + "ready_ms": 2489.014, + "upload_ms": 133.543 + }, + { + "attachment_id": "8ff95066-f752-49c2-832a-30f6f5aa7fc9", + "ready_ms": 2484.853, + "upload_ms": 178.776 + }, + { + "attachment_id": "5ee41346-dc3b-45ab-ab40-fab1b40a4b66", + "ready_ms": 2900.885, + "upload_ms": 191.95 + }, + { + "attachment_id": "689cfe15-7c9d-4917-a09c-67f7009e8f07", + "ready_ms": 2953.14, + "upload_ms": 146.877 + }, + { + "attachment_id": "f72ee233-7d79-4d9f-9dd8-2a1914974fd6", + "ready_ms": 2976.284, + "upload_ms": 143.164 + }, + { + "attachment_id": "60de5aea-2dc6-4e9e-9834-cde5d7da064a", + "ready_ms": 2954.715, + "upload_ms": 183.028 + }, + { + "attachment_id": "8c2bb984-9c2c-4c1f-9da7-d0ff68f4a6a2", + "ready_ms": 4023.765, + "upload_ms": 187.25 + }, + { + "attachment_id": "0dd7d082-238b-4865-b1cb-02424d3d783e", + "ready_ms": 4064.043, + "upload_ms": 182.425 + }, + { + "attachment_id": "82a36d66-a13f-415b-b54d-e900af9997c2", + "ready_ms": 4057.21, + "upload_ms": 183.918 + }, + { + "attachment_id": "8f59adc2-b335-42b5-844e-93c1fa2526ed", + "ready_ms": 4031.183, + "upload_ms": 193.777 + }, + { + "attachment_id": "cd599cbe-b96b-4d28-8120-0ab9a8e90242", + "ready_ms": 4453.48, + "upload_ms": 172.33 + }, + { + "attachment_id": "d8ca0870-e258-4e7e-a244-ffa672e85bdb", + "ready_ms": 4426.811, + "upload_ms": 148.3 + }, + { + "attachment_id": "35abe4b6-19fd-437c-befc-1dc4d4d41cf8", + "ready_ms": 4497.526, + "upload_ms": 186.67 + } + ] + }, + { + "concurrency": 1, + "prefetch": 1, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 25.854, + "throughput_rps": 0.774, + "ready_p50_ms": 12939.763, + "ready_p95_ms": 22792.626, + "after_accept_p95_ms": 21836.999, + "upload_p95_ms": 1073.912, + "queue_lag_peak_seconds": 21.476154, + "outstanding_peak": 20, + "rss_baseline_bytes": 15982592, + "rss_peak_bytes": 77754368, + "samples": [ + { + "seconds": 1.02, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 33550336 + }, + { + "seconds": 1.8, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 0.688855, + "rss_bytes": 53231616 + }, + { + "seconds": 2.378, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 1.426091, + "rss_bytes": 53948416 + }, + { + "seconds": 2.879, + "outstanding": 20, + "queued": 19, + "processing": 1, + "queue_lag_seconds": 1.947211, + "rss_bytes": 16060416 + }, + { + "seconds": 3.423, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 2.417889, + "rss_bytes": 53731328 + }, + { + "seconds": 4.011, + "outstanding": 19, + "queued": 18, + "processing": 1, + "queue_lag_seconds": 2.991099, + "rss_bytes": 74772480 + }, + { + "seconds": 4.552, + "outstanding": 18, + "queued": 17, + "processing": 1, + "queue_lag_seconds": 3.494125, + "rss_bytes": 53624832 + }, + { + "seconds": 5.073, + "outstanding": 18, + "queued": 17, + "processing": 1, + "queue_lag_seconds": 4.010811, + "rss_bytes": 74854400 + }, + { + "seconds": 5.596, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 4.539016, + "rss_bytes": 53174272 + }, + { + "seconds": 6.251, + "outstanding": 17, + "queued": 16, + "processing": 1, + "queue_lag_seconds": 5.130687, + "rss_bytes": 64868352 + }, + { + "seconds": 6.763, + "outstanding": 16, + "queued": 15, + "processing": 1, + "queue_lag_seconds": 5.607154, + "rss_bytes": 48951296 + }, + { + "seconds": 7.335, + "outstanding": 16, + "queued": 15, + "processing": 1, + "queue_lag_seconds": 6.118839, + "rss_bytes": 60424192 + }, + { + "seconds": 7.84, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 6.573697, + "rss_bytes": 50601984 + }, + { + "seconds": 8.348, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 7.107352, + "rss_bytes": 53428224 + }, + { + "seconds": 8.827, + "outstanding": 15, + "queued": 14, + "processing": 1, + "queue_lag_seconds": 7.583777, + "rss_bytes": 16121856 + }, + { + "seconds": 9.337, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 7.987629, + "rss_bytes": 52092928 + }, + { + "seconds": 9.826, + "outstanding": 14, + "queued": 13, + "processing": 1, + "queue_lag_seconds": 8.50605, + "rss_bytes": 64462848 + }, + { + "seconds": 10.355, + "outstanding": 13, + "queued": 12, + "processing": 1, + "queue_lag_seconds": 8.956849, + "rss_bytes": 50950144 + }, + { + "seconds": 10.835, + "outstanding": 13, + "queued": 12, + "processing": 1, + "queue_lag_seconds": 9.439881, + "rss_bytes": 57647104 + }, + { + "seconds": 11.36, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 9.732071, + "rss_bytes": 48742400 + }, + { + "seconds": 11.899, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 10.240891, + "rss_bytes": 53321728 + }, + { + "seconds": 12.396, + "outstanding": 12, + "queued": 11, + "processing": 1, + "queue_lag_seconds": 10.797879, + "rss_bytes": 16224256 + }, + { + "seconds": 12.877, + "outstanding": 11, + "queued": 10, + "processing": 1, + "queue_lag_seconds": 11.191622, + "rss_bytes": 53665792 + }, + { + "seconds": 13.351, + "outstanding": 11, + "queued": 10, + "processing": 1, + "queue_lag_seconds": 11.66977, + "rss_bytes": 77754368 + }, + { + "seconds": 13.824, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 11.950794, + "rss_bytes": 52498432 + }, + { + "seconds": 14.358, + "outstanding": 10, + "queued": 9, + "processing": 1, + "queue_lag_seconds": 12.498662, + "rss_bytes": 57901056 + }, + { + "seconds": 14.837, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 12.758517, + "rss_bytes": 47919104 + }, + { + "seconds": 15.313, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 13.243371, + "rss_bytes": 53309440 + }, + { + "seconds": 15.775, + "outstanding": 9, + "queued": 8, + "processing": 1, + "queue_lag_seconds": 13.706933, + "rss_bytes": 62353408 + }, + { + "seconds": 16.325, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 14.200687, + "rss_bytes": 53628928 + }, + { + "seconds": 16.786, + "outstanding": 8, + "queued": 7, + "processing": 1, + "queue_lag_seconds": 14.666856, + "rss_bytes": 70582272 + }, + { + "seconds": 17.278, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 15.03883, + "rss_bytes": 49065984 + }, + { + "seconds": 17.734, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 15.50651, + "rss_bytes": 52953088 + }, + { + "seconds": 18.344, + "outstanding": 7, + "queued": 6, + "processing": 1, + "queue_lag_seconds": 16.077438, + "rss_bytes": 16084992 + }, + { + "seconds": 18.792, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 16.479718, + "rss_bytes": 53477376 + }, + { + "seconds": 19.245, + "outstanding": 6, + "queued": 5, + "processing": 1, + "queue_lag_seconds": 16.930317, + "rss_bytes": 76738560 + }, + { + "seconds": 19.695, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 17.31214, + "rss_bytes": 48459776 + }, + { + "seconds": 20.195, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 17.799775, + "rss_bytes": 60489728 + }, + { + "seconds": 20.648, + "outstanding": 5, + "queued": 4, + "processing": 1, + "queue_lag_seconds": 18.259385, + "rss_bytes": 15990784 + }, + { + "seconds": 21.143, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 18.717576, + "rss_bytes": 53784576 + }, + { + "seconds": 21.606, + "outstanding": 4, + "queued": 3, + "processing": 1, + "queue_lag_seconds": 19.175829, + "rss_bytes": 73236480 + }, + { + "seconds": 22.058, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 19.563274, + "rss_bytes": 52822016 + }, + { + "seconds": 22.516, + "outstanding": 3, + "queued": 2, + "processing": 1, + "queue_lag_seconds": 20.017596, + "rss_bytes": 66035712 + }, + { + "seconds": 23.125, + "outstanding": 2, + "queued": 2, + "processing": 0, + "queue_lag_seconds": 20.531987, + "rss_bytes": 47538176 + }, + { + "seconds": 23.606, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 21.033856, + "rss_bytes": 53334016 + }, + { + "seconds": 24.05, + "outstanding": 2, + "queued": 1, + "processing": 1, + "queue_lag_seconds": 21.476154, + "rss_bytes": 66752512 + }, + { + "seconds": 24.52, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 51838976 + }, + { + "seconds": 24.956, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 63279104 + }, + { + "seconds": 25.408, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 16261120 + }, + { + "seconds": 25.854, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16261120 + } + ], + "observations": [ + { + "attachment_id": "72ff9bf3-1dff-4644-a4d6-ae4bd7a617a8", + "ready_ms": 2978.062, + "upload_ms": 489.72 + }, + { + "attachment_id": "abe6a64b-f1e5-40d5-8ceb-3a7da3999640", + "ready_ms": 4113.182, + "upload_ms": 582.949 + }, + { + "attachment_id": "1dadde6d-8831-4035-91d1-00dd21897af4", + "ready_ms": 5167.355, + "upload_ms": 651.946 + }, + { + "attachment_id": "408e49cd-a346-42b2-8c8c-9ef1613ff231", + "ready_ms": 6855.691, + "upload_ms": 771.882 + }, + { + "attachment_id": "8277e968-cb19-42b4-9d1d-99d4221e1545", + "ready_ms": 7948.368, + "upload_ms": 855.385 + }, + { + "attachment_id": "e26f1421-f2a4-4427-a07e-af265f8f0b65", + "ready_ms": 8890.393, + "upload_ms": 944.604 + }, + { + "attachment_id": "e19e3655-8a95-4379-b94f-224ec7b3855e", + "ready_ms": 9851.15, + "upload_ms": 1016.483 + }, + { + "attachment_id": "6ef71b51-44a7-4423-b9e3-89541b97e970", + "ready_ms": 11385.961, + "upload_ms": 1105.276 + }, + { + "attachment_id": "110f5859-5437-4fc5-90c9-81791efeab6b", + "ready_ms": 11913.871, + "upload_ms": 658.419 + }, + { + "attachment_id": "c8a362ec-ea03-4cd2-8eba-3438a6debef2", + "ready_ms": 12939.763, + "upload_ms": 850.829 + }, + { + "attachment_id": "64ded537-4b0a-4745-b4dc-3900ca79c812", + "ready_ms": 14150.284, + "upload_ms": 670.543 + }, + { + "attachment_id": "bf549960-5414-4d26-96f3-dd6431f6a987", + "ready_ms": 15205.165, + "upload_ms": 1006.479 + }, + { + "attachment_id": "cb4c3dbb-75b0-458e-873a-69fec2a480e4", + "ready_ms": 15905.145, + "upload_ms": 891.157 + }, + { + "attachment_id": "14436df8-d3da-4b50-8e68-7b724da9be51", + "ready_ms": 17584.614, + "upload_ms": 1045.919 + }, + { + "attachment_id": "0ed68cb8-e3e6-45c2-9773-bf934db75454", + "ready_ms": 18110.868, + "upload_ms": 791.025 + }, + { + "attachment_id": "30515ce8-61dc-4211-ad60-f09c2da702dd", + "ready_ms": 19659.633, + "upload_ms": 1046.097 + }, + { + "attachment_id": "cca7e14a-127d-4deb-8e52-7dc87a7ad36b", + "ready_ms": 20262.643, + "upload_ms": 750.774 + }, + { + "attachment_id": "7a185ba2-5492-45d6-bfcf-ddd7fdaf04d1", + "ready_ms": 22053.501, + "upload_ms": 1073.912 + }, + { + "attachment_id": "978ae03a-8e42-4bdf-9beb-9ae487f82651", + "ready_ms": 22792.626, + "upload_ms": 955.627 + }, + { + "attachment_id": "c428751a-9273-415a-86d7-2f7f69d0664b", + "ready_ms": 23836.824, + "upload_ms": 721.881 + } + ] + }, + { + "concurrency": 2, + "prefetch": 4, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 23.508, + "throughput_rps": 0.851, + "ready_p50_ms": 12230.053, + "ready_p95_ms": 21471.061, + "after_accept_p95_ms": 20489.723, + "upload_p95_ms": 1147.664, + "queue_lag_peak_seconds": 18.202527, + "outstanding_peak": 20, + "rss_baseline_bytes": 15872000, + "rss_peak_bytes": 140812288, + "samples": [ + { + "seconds": 0.804, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 15970304 + }, + { + "seconds": 1.609, + "outstanding": 7, + "queued": 5, + "processing": 2, + "queue_lag_seconds": 0.522618, + "rss_bytes": 85676032 + }, + { + "seconds": 2.335, + "outstanding": 13, + "queued": 11, + "processing": 2, + "queue_lag_seconds": 1.388412, + "rss_bytes": 91406336 + }, + { + "seconds": 2.964, + "outstanding": 20, + "queued": 18, + "processing": 2, + "queue_lag_seconds": 2.023378, + "rss_bytes": 102551552 + }, + { + "seconds": 3.531, + "outstanding": 20, + "queued": 18, + "processing": 2, + "queue_lag_seconds": 2.574746, + "rss_bytes": 62296064 + }, + { + "seconds": 4.141, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 2.974395, + "rss_bytes": 86925312 + }, + { + "seconds": 4.741, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 3.553895, + "rss_bytes": 90320896 + }, + { + "seconds": 5.322, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 4.137822, + "rss_bytes": 117186560 + }, + { + "seconds": 6.059, + "outstanding": 18, + "queued": 16, + "processing": 2, + "queue_lag_seconds": 4.877053, + "rss_bytes": 24940544 + }, + { + "seconds": 6.63, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 5.248695, + "rss_bytes": 91217920 + }, + { + "seconds": 7.249, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 5.841495, + "rss_bytes": 91856896 + }, + { + "seconds": 7.835, + "outstanding": 16, + "queued": 14, + "processing": 2, + "queue_lag_seconds": 6.43863, + "rss_bytes": 140812288 + }, + { + "seconds": 8.424, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 6.719826, + "rss_bytes": 79773696 + }, + { + "seconds": 9.003, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 7.268924, + "rss_bytes": 93224960 + }, + { + "seconds": 9.723, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 7.961311, + "rss_bytes": 109162496 + }, + { + "seconds": 10.343, + "outstanding": 14, + "queued": 12, + "processing": 2, + "queue_lag_seconds": 8.675691, + "rss_bytes": 66420736 + }, + { + "seconds": 10.913, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 8.881136, + "rss_bytes": 89284608 + }, + { + "seconds": 11.473, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 9.410813, + "rss_bytes": 92332032 + }, + { + "seconds": 12.095, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 10.044173, + "rss_bytes": 120737792 + }, + { + "seconds": 12.639, + "outstanding": 12, + "queued": 10, + "processing": 2, + "queue_lag_seconds": 10.643923, + "rss_bytes": 47562752 + }, + { + "seconds": 13.156, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 10.865062, + "rss_bytes": 90583040 + }, + { + "seconds": 13.667, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 11.419681, + "rss_bytes": 93134848 + }, + { + "seconds": 14.213, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 11.932928, + "rss_bytes": 117977088 + }, + { + "seconds": 14.717, + "outstanding": 10, + "queued": 8, + "processing": 2, + "queue_lag_seconds": 12.476196, + "rss_bytes": 16855040 + }, + { + "seconds": 15.259, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 12.747498, + "rss_bytes": 89051136 + }, + { + "seconds": 15.805, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 13.312622, + "rss_bytes": 92135424 + }, + { + "seconds": 16.369, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 13.827427, + "rss_bytes": 119685120 + }, + { + "seconds": 16.916, + "outstanding": 8, + "queued": 6, + "processing": 2, + "queue_lag_seconds": 14.41196, + "rss_bytes": 61480960 + }, + { + "seconds": 17.474, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 14.838173, + "rss_bytes": 91725824 + }, + { + "seconds": 17.982, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 15.313376, + "rss_bytes": 93421568 + }, + { + "seconds": 18.543, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 15.910134, + "rss_bytes": 125739008 + }, + { + "seconds": 19.008, + "outstanding": 6, + "queued": 4, + "processing": 2, + "queue_lag_seconds": 16.389491, + "rss_bytes": 16818176 + }, + { + "seconds": 19.538, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 16.706092, + "rss_bytes": 89124864 + }, + { + "seconds": 20.023, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 17.209332, + "rss_bytes": 91959296 + }, + { + "seconds": 20.522, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 17.719409, + "rss_bytes": 108560384 + }, + { + "seconds": 21.036, + "outstanding": 4, + "queued": 2, + "processing": 2, + "queue_lag_seconds": 18.202527, + "rss_bytes": 69586944 + }, + { + "seconds": 21.526, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 86437888 + }, + { + "seconds": 22.05, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 92631040 + }, + { + "seconds": 22.568, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 112336896 + }, + { + "seconds": 23.064, + "outstanding": 2, + "queued": 0, + "processing": 2, + "queue_lag_seconds": 0, + "rss_bytes": 78319616 + }, + { + "seconds": 23.507, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 16228352 + } + ], + "observations": [ + { + "attachment_id": "99688333-40bf-4d41-a4fe-323fad0904e9", + "ready_ms": 3560.499, + "upload_ms": 408.011 + }, + { + "attachment_id": "f147a922-d1fc-42cc-a66b-142d571f82df", + "ready_ms": 3554.025, + "upload_ms": 457.303 + }, + { + "attachment_id": "7658c2fe-3bb2-4ca4-8412-d0eb20a2b8ef", + "ready_ms": 6052.072, + "upload_ms": 679.643 + }, + { + "attachment_id": "89f86138-8fcd-4105-9892-c9dfe9dbecdc", + "ready_ms": 6066.401, + "upload_ms": 583.962 + }, + { + "attachment_id": "ceb8fe8c-a841-495e-bd34-5aad13dd6d75", + "ready_ms": 8403.356, + "upload_ms": 771.179 + }, + { + "attachment_id": "bf6a320d-9cfd-4029-a1d5-6ffd3c558aed", + "ready_ms": 8440.405, + "upload_ms": 887.69 + }, + { + "attachment_id": "3ac7646a-b9ca-4e27-8aa3-763ac97e994b", + "ready_ms": 10349.01, + "upload_ms": 981.57 + }, + { + "attachment_id": "00c411bc-e7e7-4ac4-90d9-2fa2d2c98a1c", + "ready_ms": 10339.33, + "upload_ms": 1076.033 + }, + { + "attachment_id": "71e1a5b8-497c-4c4e-958e-28e0ae2ca882", + "ready_ms": 12230.053, + "upload_ms": 833.319 + }, + { + "attachment_id": "821c31eb-38ce-4a1c-9955-e9fcc2859ff8", + "ready_ms": 12176.461, + "upload_ms": 1020.951 + }, + { + "attachment_id": "ae840ffa-f8c9-4fbc-b7bc-bd59ba5e9e72", + "ready_ms": 14133.905, + "upload_ms": 1012.104 + }, + { + "attachment_id": "5ffb85c0-0441-46c8-9f39-21b3138ce662", + "ready_ms": 14053.062, + "upload_ms": 1005.261 + }, + { + "attachment_id": "70dcb558-6257-4a9d-83bb-1494a55a9ef2", + "ready_ms": 16140.026, + "upload_ms": 1082.357 + }, + { + "attachment_id": "4e0240ab-c8c8-494d-8bcc-1c43cd447738", + "ready_ms": 15936.966, + "upload_ms": 985.338 + }, + { + "attachment_id": "8d0b3749-7ace-412c-9b19-e8a28eabb84f", + "ready_ms": 18123.235, + "upload_ms": 1204.862 + }, + { + "attachment_id": "d1236a0d-6869-4930-a6a2-e90ede460170", + "ready_ms": 17782.071, + "upload_ms": 926.795 + }, + { + "attachment_id": "acfc46ce-c102-4b10-b483-c820ce2dc416", + "ready_ms": 19926.281, + "upload_ms": 1147.664 + }, + { + "attachment_id": "7b22deab-2e71-4a84-92bb-135d3b5f2fbf", + "ready_ms": 19347.044, + "upload_ms": 652.552 + }, + { + "attachment_id": "c35b9a9b-1241-4a6d-895a-99773d78f318", + "ready_ms": 21471.061, + "upload_ms": 981.339 + }, + { + "attachment_id": "a83f78fc-4201-4da9-8519-1bdcf403aa21", + "ready_ms": 21587.644, + "upload_ms": 936.535 + } + ] + }, + { + "concurrency": 4, + "prefetch": 8, + "width": 1536, + "height": 1536, + "size_bytes": 7081647, + "input_sha256": "015b73f610b2145516211f8039594f43e64dfdaf0326b2874b294727f114de02", + "offered": 20, + "accepted": 20, + "rejected": 0, + "completed": 20, + "makespan_seconds": 23.602, + "throughput_rps": 0.847, + "ready_p50_ms": 13459.561, + "ready_p95_ms": 20744.497, + "after_accept_p95_ms": 19885.435, + "upload_p95_ms": 1134.032, + "queue_lag_peak_seconds": 16.602282, + "outstanding_peak": 20, + "rss_baseline_bytes": 16134144, + "rss_peak_bytes": 235130880, + "samples": [ + { + "seconds": 0.813, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 45719552 + }, + { + "seconds": 1.493, + "outstanding": 6, + "queued": 2, + "processing": 4, + "queue_lag_seconds": 0.174778, + "rss_bytes": 143278080 + }, + { + "seconds": 2.202, + "outstanding": 13, + "queued": 9, + "processing": 4, + "queue_lag_seconds": 0.968807, + "rss_bytes": 155238400 + }, + { + "seconds": 2.933, + "outstanding": 19, + "queued": 15, + "processing": 4, + "queue_lag_seconds": 1.613169, + "rss_bytes": 165273600 + }, + { + "seconds": 3.638, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 2.323971, + "rss_bytes": 168144896 + }, + { + "seconds": 4.287, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 3.036557, + "rss_bytes": 182329344 + }, + { + "seconds": 4.921, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 3.659427, + "rss_bytes": 209608704 + }, + { + "seconds": 5.538, + "outstanding": 20, + "queued": 16, + "processing": 4, + "queue_lag_seconds": 4.347039, + "rss_bytes": 208084992 + }, + { + "seconds": 6.12, + "outstanding": 17, + "queued": 14, + "processing": 3, + "queue_lag_seconds": 4.708041, + "rss_bytes": 146980864 + }, + { + "seconds": 6.721, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 5.034121, + "rss_bytes": 155770880 + }, + { + "seconds": 7.302, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 5.617518, + "rss_bytes": 165101568 + }, + { + "seconds": 7.836, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 6.172094, + "rss_bytes": 169320448 + }, + { + "seconds": 8.419, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 6.723611, + "rss_bytes": 184619008 + }, + { + "seconds": 9.004, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 7.310578, + "rss_bytes": 225910784 + }, + { + "seconds": 9.585, + "outstanding": 16, + "queued": 12, + "processing": 4, + "queue_lag_seconds": 7.892007, + "rss_bytes": 0 + }, + { + "seconds": 10.185, + "outstanding": 14, + "queued": 10, + "processing": 4, + "queue_lag_seconds": 8.34158, + "rss_bytes": 144318464 + }, + { + "seconds": 10.798, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 8.711244, + "rss_bytes": 154238976 + }, + { + "seconds": 11.505, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 9.414926, + "rss_bytes": 165646336 + }, + { + "seconds": 12.073, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 9.9939, + "rss_bytes": 178364416 + }, + { + "seconds": 12.684, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 10.584393, + "rss_bytes": 196341760 + }, + { + "seconds": 13.288, + "outstanding": 12, + "queued": 8, + "processing": 4, + "queue_lag_seconds": 11.205643, + "rss_bytes": 200409088 + }, + { + "seconds": 13.831, + "outstanding": 11, + "queued": 7, + "processing": 4, + "queue_lag_seconds": 11.641966, + "rss_bytes": 150261760 + }, + { + "seconds": 14.407, + "outstanding": 10, + "queued": 6, + "processing": 4, + "queue_lag_seconds": 12.033763, + "rss_bytes": 193028096 + }, + { + "seconds": 15.067, + "outstanding": 9, + "queued": 5, + "processing": 4, + "queue_lag_seconds": 12.521832, + "rss_bytes": 176140288 + }, + { + "seconds": 15.668, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 13.11874, + "rss_bytes": 158498816 + }, + { + "seconds": 16.187, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 13.66311, + "rss_bytes": 170463232 + }, + { + "seconds": 16.776, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 14.235394, + "rss_bytes": 197406720 + }, + { + "seconds": 17.333, + "outstanding": 8, + "queued": 4, + "processing": 4, + "queue_lag_seconds": 14.829522, + "rss_bytes": 196517888 + }, + { + "seconds": 17.899, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 15.288305, + "rss_bytes": 209166336 + }, + { + "seconds": 18.463, + "outstanding": 7, + "queued": 3, + "processing": 4, + "queue_lag_seconds": 15.737111, + "rss_bytes": 130351104 + }, + { + "seconds": 18.99, + "outstanding": 5, + "queued": 1, + "processing": 4, + "queue_lag_seconds": 16.061496, + "rss_bytes": 180559872 + }, + { + "seconds": 19.488, + "outstanding": 5, + "queued": 1, + "processing": 4, + "queue_lag_seconds": 16.602282, + "rss_bytes": 184385536 + }, + { + "seconds": 20.008, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 159518720 + }, + { + "seconds": 20.47, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 175730688 + }, + { + "seconds": 20.953, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 184320000 + }, + { + "seconds": 21.55, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 235130880 + }, + { + "seconds": 22.051, + "outstanding": 4, + "queued": 0, + "processing": 4, + "queue_lag_seconds": 0, + "rss_bytes": 111206400 + }, + { + "seconds": 22.708, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 59723776 + }, + { + "seconds": 23.178, + "outstanding": 1, + "queued": 0, + "processing": 1, + "queue_lag_seconds": 0, + "rss_bytes": 17543168 + }, + { + "seconds": 23.601, + "outstanding": 0, + "queued": 0, + "processing": 0, + "queue_lag_seconds": 0, + "rss_bytes": 17477632 + } + ], + "observations": [ + { + "attachment_id": "09e001f8-6730-41fd-bd5c-0105e0a519a1", + "ready_ms": 5635.479, + "upload_ms": 379.774 + }, + { + "attachment_id": "fa2633da-3cfe-4d74-b345-d01ecee4c603", + "ready_ms": 6197.281, + "upload_ms": 811.621 + }, + { + "attachment_id": "06ab46de-9fe5-4294-8667-3697727ec660", + "ready_ms": 6192.099, + "upload_ms": 643.883 + }, + { + "attachment_id": "8b125e8a-db3f-48fc-a2a6-70fc90f0da7d", + "ready_ms": 6211.204, + "upload_ms": 465.159 + }, + { + "attachment_id": "7b8a17f9-71cc-452d-b05b-b04a109e0f86", + "ready_ms": 9635.43, + "upload_ms": 1295.364 + }, + { + "attachment_id": "3a216daa-9184-4143-aa24-c2fb26fe65b7", + "ready_ms": 9654.948, + "upload_ms": 898.299 + }, + { + "attachment_id": "3428eb34-cd42-4c1c-9bab-5733561fd44a", + "ready_ms": 10266.06, + "upload_ms": 1051.242 + }, + { + "attachment_id": "ff49cac4-62c3-4005-b114-71f116d51d29", + "ready_ms": 10847.423, + "upload_ms": 1134.032 + }, + { + "attachment_id": "9fae56a4-ba64-4947-809e-34c10051205c", + "ready_ms": 12577.222, + "upload_ms": 751.709 + }, + { + "attachment_id": "c627cf28-b4e7-42e9-bdf9-83d6ef9f780b", + "ready_ms": 13459.561, + "upload_ms": 949.39 + }, + { + "attachment_id": "cd56a914-37d4-4b51-bb19-60f6a3378ced", + "ready_ms": 14769.857, + "upload_ms": 1113.794 + }, + { + "attachment_id": "9ff00757-950e-4fad-83ab-f1fecbc97cfa", + "ready_ms": 15101.025, + "upload_ms": 1013.618 + }, + { + "attachment_id": "928fadf0-75c4-45ce-9718-91d7101cf7c5", + "ready_ms": 16911.15, + "upload_ms": 739.552 + }, + { + "attachment_id": "da2cbbd5-3afa-42d0-9a37-2b7fbe6629dc", + "ready_ms": 17620.17, + "upload_ms": 1059.124 + }, + { + "attachment_id": "973b6abd-880b-49d3-accd-101d9cc87c3e", + "ready_ms": 17220.489, + "upload_ms": 772.069 + }, + { + "attachment_id": "9ff48419-a077-4b62-a0dc-f779ccd7f87b", + "ready_ms": 18925.466, + "upload_ms": 1036.736 + }, + { + "attachment_id": "45f54fd6-1c3c-44e8-9a69-f34f28e31be5", + "ready_ms": 20580.767, + "upload_ms": 858.08 + }, + { + "attachment_id": "36296a7a-7fda-4b12-b026-c6fc433d9753", + "ready_ms": 20744.497, + "upload_ms": 933.986 + }, + { + "attachment_id": "ceefd3ec-372a-4264-a35a-7944b2717699", + "ready_ms": 20668.569, + "upload_ms": 783.134 + }, + { + "attachment_id": "0389dd0c-e1f0-4c01-a5ad-08c3e93af110", + "ready_ms": 21584.932, + "upload_ms": 901.456 + } + ] + } + ], + "overload": { + "limit": 8, + "offered": 16, + "accepted": 8, + "rejected_429": 8, + "observed_api_instances": 2, + "outstanding_worker_stopped": 8, + "ready_after_resume": 8, + "observed_drain_after_worker_healthy_seconds": 0.062, + "outstanding_after_recovery": 0, + "followup_upload": 202, + "text_messages_during_overload": "PASS" + }, + "result": "PASS", + "restored_initial_settings": true, + "finished_at": "2026-11-10T19:08:13.363296+00:00" +} diff --git a/evidence/lab5-research.png b/evidence/lab5-research.png new file mode 100644 index 00000000..aa5eaaab Binary files /dev/null and b/evidence/lab5-research.png differ diff --git a/evidence/lab5-research.svg b/evidence/lab5-research.svg new file mode 100644 index 00000000..27ec2d8c --- /dev/null +++ b/evidence/lab5-research.svg @@ -0,0 +1,2752 @@ + + + + + + + + 2026-11-10T22:08:55.028824 + image/svg+xml + + + Matplotlib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/lab5-smoke.txt b/evidence/lab5-smoke.txt new file mode 100644 index 00000000..c1bf3fc7 --- /dev/null +++ b/evidence/lab5-smoke.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/smoke.py --base-url "http://localhost:18085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5-validation.txt b/evidence/lab5-validation.txt new file mode 100644 index 00000000..3f7f5878 --- /dev/null +++ b/evidence/lab5-validation.txt @@ -0,0 +1,19 @@ +Fresh lab5 validation, 2026-11-10. +Root independent checks: + make check validate: exit 0. + go vet ./...: exit 0. + ./scripts/compose.sh config --quiet: exit 0. + docker stack config -c stack.yaml: exit 0. + sh -n scripts/*.sh infra/*.sh: exit 0. +Worker checks: + go test -race ./...: exit 0. + make test: exit 0, lab5 scenario on port18085. + make test-integration: exit 0 on repeat, including 005-010 up/down-to4/legacy/reset/up. + Integration initial failure was an inherited fixture that dropped the migration journal from an already-lab5 schema. + The fixture now models an actual lab4 schema (down-to4) before legacy-journal adoption. + Fault cases: all 11 groups verified across the initial main phase and --tail-only repeat. + The original full invocation exited 1 due to a test stop-helper race; it is not claimed as a full green run. + make test-persistence: exit 0; own stack restart preserved confirmed data and existing session. + make test-failover: exit 0; one of two API replicas stopped, 123 successful writes, 1 failed attempt, recovery 0.696s. +Public contracts and tests match skeleton276dbda; optional private tests cover added requirements. +Historical evidence/lab4-* is not used as current lab5 evidence. diff --git a/infra/README.md b/infra/README.md index 82e02813..3504ad78 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,14 +1,24 @@ -# Инфраструктура студента +# Инфраструктура lab5 -Основной стенд использует `stack.yaml`: HAProxy → две API-реплики → PostgreSQL, Redis AOF и приватный MinIO. -Запуск: `make up`. Остановка с сохранением данных: `make down`. -Версии, секреты, ограничения и проверки описаны в [REPORT.md](../REPORT.md). +HAProxy направляет HTTP в две API-реплики. +PostgreSQL хранит сообщения, метаданные, jobs, outbox и audit replay. +Redis хранит сессии и Streams с consumer group `image-workers-v1`. +Redis использует AOF и `appendfsync always`; Pub/Sub не используется. +MinIO хранит приватные staging и конечные варианты. Worker работает отдельным сервисом. -`s3-bootstrap.sh` идемпотентно создаёт bucket и отдельного API-пользователя по `s3-policy.json`. -`minio-entrypoint.sh` читает root keys из Docker secrets. Консоль слушает loopback контейнера. -`Dockerfile.storage` собирает MinIO и mc из исходников с фиксированными commit и SHA-256. -`haproxy.cfg` проверяет readiness реплик. `redis.conf` включает AOF и запрещает eviction. -`migrate.sh` запускает goose без автоматической загрузки файлов окружения. +Основной запуск: `make up`. API: http://localhost:18085. +Остановка с сохранением volumes: `make down`. +Compose запускает те же роли с одной API-репликой. +Скрипты не читают `.env`; bootstrap создаёт Docker secrets. +Redis имеет лимит памяти 384MiB и `maxmemory 256mb`, политика `noeviction`. +Worker имеет лимит 1CPU/1536MiB; API — по 1CPU/1536MiB. -Restart hook перезапускает API, HAProxy, PostgreSQL, Redis и MinIO с сохранением volumes. -Failover hook останавливает API-реплику по `TARGET_INSTANCE_ID`; Swarm создаёт новую task с другим ID. +Миграции goose 005–010 добавляют состояния, jobs, outbox, audit и cleanup. +Миграция 009 переводит прежние оригиналы в устойчивый staging без потери байтов. +При обновлении API останавливаются перед миграциями, затем запускается новая версия. +Swarm configs Redis и S3 policy используют суффикс `_v5`. +Это позволяет обновить прежний стенд без изменения immutable configs. + +Порты PostgreSQL, Redis, MinIO и worker не опубликованы. +S3 policy разрешает API/worker только `original/*` и `staging/*` своего bucket. +Версии образов, протокол отказов и команды находятся в [REPORT.md](../REPORT.md). diff --git a/infra/redis.conf b/infra/redis.conf index c9e470c6..0b8eb4c4 100644 --- a/infra/redis.conf +++ b/infra/redis.conf @@ -5,5 +5,5 @@ dir /data appendonly yes appendfsync always save "" -maxmemory 64mb +maxmemory 256mb maxmemory-policy noeviction diff --git a/infra/s3-policy.json b/infra/s3-policy.json index b9fb3b95..20797ca1 100644 --- a/infra/s3-policy.json +++ b/infra/s3-policy.json @@ -10,12 +10,12 @@ "Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::attachments"], - "Condition": {"StringLike": {"s3:prefix": ["original/*"]}} + "Condition": {"StringLike": {"s3:prefix": ["original/*", "staging/*"]}} }, { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": ["arn:aws:s3:::attachments/original/*"] + "Resource": ["arn:aws:s3:::attachments/original/*", "arn:aws:s3:::attachments/staging/*"] } ] } diff --git a/internal/app/chat/attachments.go b/internal/app/chat/attachments.go index f326d707..c345e177 100644 --- a/internal/app/chat/attachments.go +++ b/internal/app/chat/attachments.go @@ -33,7 +33,7 @@ func (h *Chat) uploadAttachment(c echo.Context) error { if err != nil { return h.useCaseError(c, err) } - return c.JSON(http.StatusCreated, a) + return c.JSON(http.StatusAccepted, a) } func (h *Chat) getAttachment(c echo.Context) error { a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) @@ -51,8 +51,15 @@ func (h *Chat) attachmentContent(c echo.Context) error { if err != nil { return h.invalid(c, "Некорректный variant") } - if present && variant != "original" { - return h.writeError(c, 404, "not_found", "Вариант не найден") + if a.Status != "ready" { + return h.writeError(c, 409, "conflict", "Вложение не готово") + } + if !present { + variant = "original" + } + a, err = h.attachments.Variant(a, variant) + if err != nil { + return h.useCaseError(c, err) } ctx, cancel := context.WithTimeout(c.Request().Context(), 45*time.Second) defer cancel() diff --git a/internal/app/chat/errors.go b/internal/app/chat/errors.go index c556830e..7c532f81 100644 --- a/internal/app/chat/errors.go +++ b/internal/app/chat/errors.go @@ -35,6 +35,9 @@ func (h *Chat) useCaseError(c echo.Context, err error) error { switch applicationError.Code { case "unsupported_media_type": status = http.StatusUnsupportedMediaType + case "rate_limited": + status = http.StatusTooManyRequests + c.Response().Header().Set("Retry-After", "2") case "payload_too_large": status = http.StatusRequestEntityTooLarge case domain.ErrorUnavailable: diff --git a/internal/attachment/async.go b/internal/attachment/async.go new file mode 100644 index 00000000..3e9faa69 --- /dev/null +++ b/internal/attachment/async.go @@ -0,0 +1,156 @@ +package attachment + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +type Options struct { + Concurrency, Prefetch, MaxOutstanding, MaxAttempts, PixelBudget int + Timeout, Lease time.Duration +} + +func LoadOptions() (Options, error) { + o := Options{} + for _, v := range []struct { + name string + target *int + def, max int + }{ + {"WORKER_CONCURRENCY", &o.Concurrency, 2, 8}, {"WORKER_PREFETCH", &o.Prefetch, 4, 32}, + {"MAX_OUTSTANDING_JOBS", &o.MaxOutstanding, 200, 10000}, {"WORKER_MAX_ATTEMPTS", &o.MaxAttempts, 5, 20}, + {"WORKER_PIXEL_BUDGET", &o.PixelBudget, 40000000, 80000000}, + } { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var err error + n, err = strconv.Atoi(raw) + if err != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > v.max { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = n + } + for _, v := range []struct { + name string + target *time.Duration + def int + }{{"WORKER_TIMEOUT_SECONDS", &o.Timeout, 25}, {"WORKER_LEASE_SECONDS", &o.Lease, 40}} { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var e error + n, e = strconv.Atoi(raw) + if e != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > 300 { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = time.Duration(n) * time.Second + } + if o.Lease < o.Timeout+5*time.Second { + return o, fmt.Errorf("worker lease must exceed timeout by at least 5s") + } + if o.PixelBudget < int(MaxPixels) { + return o, fmt.Errorf("WORKER_PIXEL_BUDGET must allow one 40MP image") + } + return o, nil +} +func NewAsync(pool *pgxpool.Pool, store *Store, o Options) *Service { + s := New(pool, store) + s.Async = true + s.Options = o + return s +} +func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body interface{ Read([]byte) (int, error) }) (Attachment, error) { + if !s.Async { + return s.uploadSync(ctx, chatID, userID, name, contentType, body) + } + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // All API replicas serialize admission, including inserts before their COMMIT. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return Attachment{}, unavailable() + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return Attachment{}, unavailable() + } + if pending >= s.Options.MaxOutstanding { + return Attachment{}, appError("rate_limited", "Очередь заполнена; повторите загрузку позднее") + } + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "queued", Variants: []any{}, PipelineVersion: 1} + stage := "staging/" + a.ID + a.Key = effectKey(a.ID, 1, "original") + // The stage lock protects even a slow PUT from the orphan cleaner. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, stage); err != nil { + return Attachment{}, unavailable() + } + if err = s.put(ctx, stage, original.File, original.Size, original.ContentType); err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height,status,staging_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,'queued',$11) RETURNING created_at,updated_at`, a.ID, chatID, userID, name, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height, stage).Scan(&a.CreatedAt, &a.UpdatedAt) + if err != nil { + return Attachment{}, unavailable() + } + job := uuid.NewString() + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts) VALUES($1,$2,'queued',$3)`, job, a.ID, s.Options.MaxAttempts); err != nil { + return Attachment{}, unavailable() + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_outbox(job_id) VALUES($1)`, job); err != nil { + return Attachment{}, unavailable() + } + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + return a, nil +} +func effectKey(id string, version int, name string) string { + return fmt.Sprintf("original/%s/v%d/%s", id, version, name) +} +func (s *Service) Variant(a Attachment, name string) (Attachment, error) { + if name == "original" { + return a, nil + } + for _, item := range a.Variants { + raw, _ := json.Marshal(item) + var v Variant + if json.Unmarshal(raw, &v) == nil && v.Name == name { + a.Key = effectKey(a.ID, a.PipelineVersion, name) + a.ContentType = v.ContentType + return a, nil + } + } + return Attachment{}, appError(domain.ErrorNotFound, "Вариант не найден") +} diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go index 402d7b3e..755338f4 100644 --- a/internal/attachment/reconcile.go +++ b/internal/attachment/reconcile.go @@ -139,7 +139,7 @@ func (s *Service) reconcileObject(ctx context.Context, key string, cutoff time.T defer tx.Rollback(context.Background()) var exists bool r.DBQueries++ - if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)`, key).Scan(&exists); err != nil { + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR $1 LIKE 'original/'||id::text||'/v'||pipeline_version::text||'/%')`, key).Scan(&exists); err != nil { return fmt.Errorf("metadata recheck failed") } if exists { @@ -207,9 +207,12 @@ func (s *Service) reconcileMetadata(ctx context.Context, id, key string, cutoff } if apply { r.DBQueries++ - if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing' WHERE id=$1 AND status='ready'`, id); err != nil { + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing',updated_at=clock_timestamp() WHERE id=$1 AND status='ready'`, id); err != nil { return fmt.Errorf("mark missing failed") } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='failed',last_error='original_missing',fence=fence+1,updated_at=clock_timestamp() WHERE attachment_id=$1 AND state='ready'`, id); err != nil { + return fmt.Errorf("mark job missing failed") + } if err = tx.Commit(ctx); err != nil { return fmt.Errorf("commit missing failed") } diff --git a/internal/attachment/render.go b/internal/attachment/render.go new file mode 100644 index 00000000..e6cebce9 --- /dev/null +++ b/internal/attachment/render.go @@ -0,0 +1,267 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "math" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +var ErrInvalidImage = errors.New("invalid image") + +type oriented struct { + image.Image + orientation int +} + +func (o oriented) Bounds() image.Rectangle { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + if o.orientation >= 5 && o.orientation <= 8 { + w, h = h, w + } + return image.Rect(0, 0, w, h) +} +func (o oriented) At(x, y int) color.Color { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + sx, sy := x, y + switch o.orientation { + case 2: + sx = w - 1 - x + case 3: + sx, sy = w-1-x, h-1-y + case 4: + sy = h - 1 - y + case 5: + sx, sy = y, x + case 6: + sx, sy = y, h-1-x + case 7: + sx, sy = w-1-y, h-1-x + case 8: + sx, sy = w-1-y, x + } + r, g, blue, a := o.Image.At(b.Min.X+sx, b.Min.Y+sy).RGBA() + white := uint32(65535) - a + return color.RGBA{uint8((r + white) / 257), uint8((g + white) / 257), uint8((blue + white) / 257), 255} +} + +// EXIF IFD0 orientation; bounded segment parsing does not follow client offsets outside APP1. +func tiffOrientation(t []byte) int { + if len(t) < 8 { + return 1 + } + var order binary.ByteOrder + if string(t[:2]) == "II" { + order = binary.LittleEndian + } else if string(t[:2]) == "MM" { + order = binary.BigEndian + } else { + return 1 + } + if order.Uint16(t[2:4]) != 42 { + return 1 + } + offset := uint64(order.Uint32(t[4:8])) + if offset+2 > uint64(len(t)) { + return 1 + } + count := uint64(order.Uint16(t[offset : offset+2])) + for k := uint64(0); k < count; k++ { + p := offset + 2 + k*12 + if p+12 > uint64(len(t)) { + break + } + entry := t[p : p+12] + if order.Uint16(entry[:2]) == 0x112 && order.Uint16(entry[2:4]) == 3 && order.Uint32(entry[4:8]) == 1 { + v := int(order.Uint16(entry[8:10])) + if v >= 1 && v <= 8 { + return v + } + } + } + return 1 +} +func Orientation(data []byte) int { + if len(data) >= 8 && bytes.Equal(data[:8], []byte{137, 80, 78, 71, 13, 10, 26, 10}) { + for i := 8; i+12 <= len(data); { + n := uint64(binary.BigEndian.Uint32(data[i : i+4])) + if n > uint64(len(data)-i-12) { + return 1 + } + end := i + 8 + int(n) + if string(data[i+4:i+8]) == "eXIf" { + return tiffOrientation(data[i+8 : end]) + } + i = end + 4 + } + return 1 + } + if len(data) < 4 || data[0] != 255 || data[1] != 216 { + return 1 + } + for i := 2; i+4 <= len(data); { + if data[i] != 255 { + return 1 + } + for i+1 < len(data) && data[i+1] == 255 { + i++ + } + if i+3 >= len(data) { + return 1 + } + marker := data[i+1] + i += 2 + if marker == 0xda || marker == 0xd9 { + return 1 + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + n := int(binary.BigEndian.Uint16(data[i : i+2])) + if n < 2 || i+n > len(data) { + return 1 + } + body := data[i+2 : i+n] + i += n + if marker == 0xe1 && len(body) >= 14 && bytes.Equal(body[:6], []byte("Exif\x00\x00")) { + return tiffOrientation(body[6:]) + } + } + return 1 +} + +func fit(w, h, maxSide int) (int, int) { + if w <= maxSide && h <= maxSide { + return w, h + } + if w >= h { + return maxSide, max(1, int(int64(h)*int64(maxSide)/int64(w))) + } + return max(1, int(int64(w)*int64(maxSide)/int64(h))), maxSide +} + +func resize(src image.Image, region image.Rectangle, w, h int) *image.RGBA { + out := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + fy := (float64(y)+.5)*float64(region.Dy())/float64(h) - .5 + fy = math.Max(0, fy) + y0 := min(int(fy), region.Dy()-1) + y1 := min(y0+1, region.Dy()-1) + dy := fy - float64(y0) + for x := 0; x < w; x++ { + fx := (float64(x)+.5)*float64(region.Dx())/float64(w) - .5 + fx = math.Max(0, fx) + x0 := min(int(fx), region.Dx()-1) + x1 := min(x0+1, region.Dx()-1) + dx := fx - float64(x0) + colors := [4]color.RGBA{color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y1)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y1)).(color.RGBA)} + blend := func(a, b, c, d uint8) uint8 { + return uint8(math.Round(float64(a)*(1-dx)*(1-dy) + float64(b)*dx*(1-dy) + float64(c)*(1-dx)*dy + float64(d)*dx*dy)) + } + out.SetRGBA(x, y, color.RGBA{blend(colors[0].R, colors[1].R, colors[2].R, colors[3].R), blend(colors[0].G, colors[1].G, colors[2].G, colors[3].G), blend(colors[0].B, colors[1].B, colors[2].B, colors[3].B), 255}) + } + } + return out +} + +// Render runs only inside a deadline-killed child, never in the HTTP process. +func Render(ctx context.Context, source, dir, expectedSHA string) error { + raw, err := os.ReadFile(source) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + if int64(len(raw)) > MaxBytes || hex.EncodeToString(sum[:]) != expectedSHA { + return ErrInvalidImage + } + cfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return ErrInvalidImage + } + if os.Getenv("PIPELINE_FAILPOINT") == "renderer_timeout" { + fifo := filepath.Join(dir, "blocked-input.png") + if err = syscall.Mkfifo(fifo, 0600); err != nil { + return err + } + return exec.CommandContext(ctx, "cwebp", "-quiet", fifo, "-o", filepath.Join(dir, "blocked.webp")).Run() + } + decoded, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return ErrInvalidImage + } + src := oriented{decoded, Orientation(raw)} + bounds := src.Bounds() + tw, th := fit(bounds.Dx(), bounds.Dy(), 256) + ow, oh := fit(bounds.Dx(), bounds.Dy(), 1280) + variants := []Variant{} + for _, name := range []string{"thumbnail", "crop", "optimized"} { + w, h := tw, th + region := bounds + if name == "crop" { + w, h = 128, 128 + side := min(bounds.Dx(), bounds.Dy()) + left, top := (bounds.Dx()-side)/2, (bounds.Dy()-side)/2 + region = image.Rect(left, top, left+side, top+side) + } + if name == "optimized" { + w, h = ow, oh + } + img := resize(src, region, w, h) + path := filepath.Join(dir, name) + if name == "optimized" { + input := filepath.Join(dir, "webp-input.png") + f, e := os.Create(input) + if e != nil { + return e + } + e = png.Encode(f, img) + f.Close() + if e != nil { + return e + } + command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", path) + if err = command.Run(); err != nil { + return err + } + } else { + f, e := os.Create(path) + if e != nil { + return e + } + e = jpeg.Encode(f, img, &jpeg.Options{Quality: 85}) + f.Close() + if e != nil { + return e + } + } + data, e := os.ReadFile(path) + if e != nil { + return e + } + hash := sha256.Sum256(data) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + variants = append(variants, Variant{Name: name, ContentType: mime, Width: w, Height: h, Size: int64(len(data)), SHA256: hex.EncodeToString(hash[:])}) + } + data, err := json.Marshal(variants) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0600) +} diff --git a/internal/attachment/render_test.go b/internal/attachment/render_test.go new file mode 100644 index 00000000..a5ab246d --- /dev/null +++ b/internal/attachment/render_test.go @@ -0,0 +1,81 @@ +package attachment + +import ( + "bytes" + "encoding/binary" + "image" + "image/color" + "testing" +) + +func TestFitGeometry(t *testing.T) { + for _, c := range []struct{ w, h, limit, x, y int }{{320, 200, 256, 256, 160}, {513, 257, 256, 256, 128}, {257, 513, 256, 128, 256}, {12, 8, 256, 12, 8}, {1, 40000, 256, 1, 256}} { + w, h := fit(c.w, c.h, c.limit) + if w != c.x || h != c.y { + t.Fatalf("fit(%d,%d)=%d,%d", c.w, c.h, w, h) + } + } +} +func TestOrientationsAndAlpha(t *testing.T) { + img := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 3; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(1 + x + y*3), A: 255}) + } + } + expected := [][]uint8{{1, 2, 3, 4, 5, 6}, {3, 2, 1, 6, 5, 4}, {6, 5, 4, 3, 2, 1}, {4, 5, 6, 1, 2, 3}, {1, 4, 2, 5, 3, 6}, {4, 1, 5, 2, 6, 3}, {6, 3, 5, 2, 4, 1}, {3, 6, 2, 5, 1, 4}} + for i, want := range expected { + src := oriented{img, i + 1} + got := []uint8{} + for y := 0; y < src.Bounds().Dy(); y++ { + for x := 0; x < src.Bounds().Dx(); x++ { + got = append(got, src.At(x, y).(color.RGBA).R) + } + } + if !bytes.Equal(got, want) { + t.Fatalf("orientation %d: %v want %v", i+1, got, want) + } + } + img.SetNRGBA(0, 0, color.NRGBA{R: 255, A: 128}) + got := oriented{img, 1}.At(0, 0).(color.RGBA) + if got != (color.RGBA{255, 127, 127, 255}) { + t.Fatalf("white alpha: %v", got) + } +} +func TestEXIFOrientation(t *testing.T) { + for _, order := range []binary.ByteOrder{binary.LittleEndian, binary.BigEndian} { + for v := 1; v <= 8; v++ { + tiff := make([]byte, 26) + copy(tiff, "II") + if order == binary.BigEndian { + copy(tiff, "MM") + } + order.PutUint16(tiff[2:], 42) + order.PutUint32(tiff[4:], 8) + order.PutUint16(tiff[8:], 1) + order.PutUint16(tiff[10:], 0x112) + order.PutUint16(tiff[12:], 3) + order.PutUint32(tiff[14:], 1) + order.PutUint16(tiff[18:], uint16(v)) + data := append([]byte{255, 216, 255, 225, 0, 34}, append([]byte("Exif\x00\x00"), tiff...)...) + if Orientation(data) != v { + t.Fatalf("EXIF value %d", v) + } + pngData := []byte{137, 80, 78, 71, 13, 10, 26, 10} + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(tiff))) + pngData = append(pngData, size...) + pngData = append(pngData, []byte("eXIf")...) + pngData = append(pngData, tiff...) + pngData = append(pngData, 0, 0, 0, 0) + if Orientation(pngData) != v { + t.Fatalf("PNG EXIF value %d", v) + } + } + } + for _, b := range [][]byte{nil, {}, {255, 216, 255, 225, 255, 255}, {255, 216, 255, 255, 225, 0, 2}} { + if Orientation(b) != 1 { + t.Fatal("malformed EXIF should be ignored") + } + } +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go index 3156afbb..fb0a441d 100644 --- a/internal/attachment/service.go +++ b/internal/attachment/service.go @@ -2,6 +2,7 @@ package attachment import ( "context" + "encoding/json" "errors" "io" "log" @@ -19,32 +20,35 @@ const UploadTimeout = 45 * time.Second const MinimumGrace = 90 * time.Second type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - Size int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []any `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Key string `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Key string `json:"-"` + PipelineVersion int `json:"-"` } type Service struct { - Pool *pgxpool.Pool - Store *Store - slots chan struct{} + Pool *pgxpool.Pool + Store *Store + slots chan struct{} + Async bool + Options Options } func New(pool *pgxpool.Pool, store *Store) *Service { return &Service{Pool: pool, Store: store, slots: make(chan struct{}, 1)} } func (s *Service) Ready(ctx context.Context) error { return s.Store.Ready(ctx) } -func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { +func (s *Service) uploadSync(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { if err := ValidateFilename(name); err != nil { return Attachment{}, err } @@ -98,17 +102,19 @@ func (s *Service) Get(ctx context.Context, id, userID string) (Attachment, error ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() a := Attachment{Variants: []any{}} - err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt) + var variants []byte + err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at,a.updated_at,a.variants,a.pipeline_version FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &variants, &a.PipelineVersion) if errors.Is(err, pgx.ErrNoRows) { return Attachment{}, appError(domain.ErrorNotFound, "Объект не найден") } if err != nil { return Attachment{}, unavailable() } - a.UpdatedAt = a.CreatedAt - // Keep the mandatory lab4 ready-only metadata schema. Failed originals are - // represented by 409; internal status remains inspectable in reconciliation. - if a.Status != "ready" { + if err = json.Unmarshal(variants, &a.Variants); err != nil { + return Attachment{}, unavailable() + } + // The legacy fixture service preserves lab4 errors; production Async exposes all states. + if a.Status != "ready" && !s.Async { return Attachment{}, appError(domain.ErrorConflict, "Оригинал вложения недоступен") } return a, nil diff --git a/internal/attachment/worker.go b/internal/attachment/worker.go new file mode 100644 index 00000000..30c72a7f --- /dev/null +++ b/internal/attachment/worker.go @@ -0,0 +1,618 @@ +package attachment + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/semaphore" +) + +const Stream = "attachments:jobs:v1" +const Group = "image-workers-v1" + +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Width int `json:"width"` + Height int `json:"height"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` +} +type claimed struct { + JobMessage + Fence int64 + Stage, Original, MIME, SHA string + Width, Height int + Size int64 +} + +func (s *Service) put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + _, err := s.Store.Client.PutObject(ctx, s.Store.Bucket, key, r, size, minio.PutObjectOptions{ContentType: mime, DisableMultipart: true}) + return err +} +func (s *Service) Publish(ctx context.Context, r *redis.Client) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT j.id::text,j.attachment_id::text,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.state='queued' ORDER BY o.available_at LIMIT 32 FOR UPDATE OF o SKIP LOCKED`) + if err != nil { + return err + } + messages := []JobMessage{} + for rows.Next() { + m := JobMessage{SchemaVersion: 1, Type: "attachment.process.v1"} + if err = rows.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt); err != nil { + rows.Close() + return err + } + m.CreatedAt = m.CreatedAt.UTC() + messages = append(messages, m) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, m := range messages { + raw, _ := json.Marshal(m) + id, e := r.XAdd(ctx, &redis.XAddArgs{Stream: Stream, Values: map[string]interface{}{"job": string(raw)}}).Result() + if e != nil { + return e + } + if _, e = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=clock_timestamp(),stream_id=$2 WHERE job_id=$1`, m.JobID, id); e != nil { + return e + } + } + return tx.Commit(ctx) +} +func (s *Service) recover(ctx context.Context) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts FROM attachment_jobs WHERE state='processing' AND lease_until= l.max { + state = "failed" + v := "retry_exhausted" + code = &v + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state=$2,lease_until=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),last_error='worker_lost' WHERE id=$1`, l.id, state); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,updated_at=clock_timestamp() WHERE id=$1`, l.att, state, code); err != nil { + return err + } + if state == "queued" { + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, l.id); err != nil { + return err + } + } + } + return tx.Commit(ctx) +} +func (s *Service) claim(ctx context.Context, m JobMessage) (claimed, bool, error) { + j := claimed{JobMessage: m} + tx, err := s.Pool.Begin(ctx) + if err != nil { + return j, false, err + } + defer tx.Rollback(context.Background()) + var state string + var attempts, max, version int + var available time.Time + err = tx.QueryRow(ctx, `SELECT state,attempts,max_attempts,available_at,pipeline_version FROM attachment_jobs WHERE id=$1 AND attachment_id=$2 FOR UPDATE`, m.JobID, m.AttachmentID).Scan(&state, &attempts, &max, &available, &version) + if errors.Is(err, pgx.ErrNoRows) { + return j, false, nil + } + if err != nil { + return j, false, err + } + if state == "ready" { + var key, mime string + var size int64 + var raw []byte + if err = tx.QueryRow(ctx, `SELECT object_key,content_type,size_bytes,variants FROM attachments WHERE id=$1 AND status='ready' AND pipeline_version=$2`, m.AttachmentID, version).Scan(&key, &mime, &size, &raw); err != nil { + return j, false, err + } + var variants []Variant + if err = json.Unmarshal(raw, &variants); err != nil || len(variants) != 3 { + return j, false, fmt.Errorf("completed effect metadata incomplete") + } + objects := map[string]int64{key: size} + for _, v := range variants { + objects[effectKey(m.AttachmentID, version, v.Name)] = v.Size + } + for object, n := range objects { + info, e := s.Store.Client.StatObject(ctx, s.Store.Bucket, object, minio.StatObjectOptions{}) + if e != nil { + return j, false, e + } + if info.Size != n { + return j, false, fmt.Errorf("completed effect size mismatch") + } + } + return j, false, nil + } + if state != "queued" || available.After(time.Now()) || m.Attempt != attempts+1 || version != m.PipelineVersion { + return j, false, nil + } + if attempts >= max { + return j, false, fmt.Errorf("attempt invariant violated") + } + err = tx.QueryRow(ctx, `UPDATE attachment_jobs SET state='processing',attempts=attempts+1,fence=fence+1,lease_until=clock_timestamp()+$2*interval '1 second',updated_at=clock_timestamp() WHERE id=$1 RETURNING fence`, m.JobID, s.Options.Lease.Seconds()).Scan(&j.Fence) + if err != nil { + return j, false, err + } + err = tx.QueryRow(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=$1 AND pipeline_version=$2 AND status IN ('queued','processing') RETURNING staging_key,object_key,content_type,size_bytes,sha256,width,height`, m.AttachmentID, version).Scan(&j.Stage, &j.Original, &j.MIME, &j.Size, &j.SHA, &j.Width, &j.Height) + if err != nil { + return j, false, err + } + return j, true, tx.Commit(ctx) +} + +// finish publishes metadata only for the current, unexpired fencing token. +func (s *Service) finish(ctx context.Context, j claimed, variants []Variant, code string, permanent bool) (bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var max int + err = tx.QueryRow(ctx, `SELECT max_attempts FROM attachment_jobs WHERE id=$1 AND fence=$2 AND state='processing' AND lease_until>clock_timestamp() AND pipeline_version=$3 FOR UPDATE`, j.JobID, j.Fence, j.PipelineVersion).Scan(&max) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + lastError := code + state := "ready" + var failure *string + available := time.Now() + if code != "" { + state = "queued" + if permanent || j.Attempt >= max { + state = "failed" + if !permanent { + code = "retry_exhausted" + } + failure = &code + } + delay := time.Duration(1< 0 { + op, cancel = context.WithTimeout(ctx, time.Second) + streams, e := r.XReadGroup(op, &redis.XReadGroupArgs{Group: Group, Consumer: consumer, Streams: []string{Stream, ">"}, Count: int64(room), Block: 100 * time.Millisecond}).Result() + cancel() + if e == nil { + for _, stream := range streams { + for _, m := range stream.Messages { + select { + case queue <- m: + case <-ctx.Done(): + return nil + } + } + } + } + } + if time.Since(lastCleanup) >= 5*time.Second { + lastCleanup = time.Now() + op, cancel = context.WithTimeout(ctx, 5*time.Second) + s.cleanStages(op) + cancel() + } + } +} +func ack(ctx context.Context, r *redis.Client, id string) { + if r.XAck(ctx, Stream, Group, id).Err() == nil { + r.XDel(ctx, Stream, id) + } +} + +// Replay is an explicit operator action, serialized with admission and job claims. +// It preserves job/effect identity and refuses already successful work. +func (s *Service) Replay(ctx context.Context, id, operator, reason string) error { + if uuid.Validate(id) != nil || len(operator) < 1 || len(operator) > 100 || len(reason) < 1 || len(reason) > 300 { + return fmt.Errorf("job UUID, operator and reason are required") + } + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return err + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return err + } + if pending >= s.Options.MaxOutstanding { + return fmt.Errorf("admission limit reached") + } + var att, state, previous string + err = tx.QueryRow(ctx, `SELECT attachment_id::text,state,coalesce(last_error,'processing_failed') FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id).Scan(&att, &state, &previous) + if err != nil { + return err + } + if state != "failed" { + return fmt.Errorf("only failed jobs may be replayed") + } + var stage *string + if err = tx.QueryRow(ctx, `SELECT staging_key FROM attachments WHERE id=$1 FOR UPDATE`, att).Scan(&stage); err != nil { + return err + } + if stage == nil { + return fmt.Errorf("staging retention expired") + } + if _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, *stage, minio.StatObjectOptions{}); err != nil { + return fmt.Errorf("staging unavailable") + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,operator,reason,previous_error) VALUES($1,$2,$3,$4)`, id, operator, reason, previous); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempts=0,fence=fence+1,last_error=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=$1`, id); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, att); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, id); err != nil { + return err + } + return tx.Commit(ctx) +} + +// Keep failed stages for seven days for explicit replay; ready stages are garbage. +func (s *Service) cleanStages(ctx context.Context) { + rows, err := s.Pool.Query(ctx, `SELECT id::text FROM attachments WHERE (status='ready' AND staging_key IS NOT NULL) OR (status='failed' AND updated_at'failed' AND error_code IS NULL)), + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text; + +-- +goose Down +UPDATE attachments SET status='failed',error_code='pipeline_downgrade' WHERE status IN ('queued','processing'); +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check CHECK (status IN ('ready','failed')), + DROP CONSTRAINT attachments_check, + ADD CONSTRAINT attachments_check CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)), + DROP COLUMN variants, DROP COLUMN updated_at, DROP COLUMN pipeline_version, DROP COLUMN staging_key; diff --git a/migrations/006_attachment_jobs.sql b/migrations/006_attachment_jobs.sql new file mode 100644 index 00000000..96fe8715 --- /dev/null +++ b/migrations/006_attachment_jobs.sql @@ -0,0 +1,18 @@ +-- +goose Up +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1, + state text NOT NULL CHECK (state IN ('queued','processing','ready','failed')), + attempts integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 20), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_until timestamptz, + fence bigint NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_jobs; diff --git a/migrations/007_attachment_outbox.sql b/migrations/007_attachment_outbox.sql new file mode 100644 index 00000000..4aa021c3 --- /dev/null +++ b/migrations/007_attachment_outbox.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE attachment_outbox ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + stream_id text +); + +-- +goose Down +DROP TABLE attachment_outbox; diff --git a/migrations/008_attachment_replays.sql b/migrations/008_attachment_replays.sql new file mode 100644 index 00000000..2587f83c --- /dev/null +++ b/migrations/008_attachment_replays.sql @@ -0,0 +1,12 @@ +-- +goose Up +CREATE TABLE attachment_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + operator text NOT NULL CHECK (char_length(operator) BETWEEN 1 AND 100), + reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 300), + previous_error text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_replays; diff --git a/migrations/009_backfill_pipeline.sql b/migrations/009_backfill_pipeline.sql new file mode 100644 index 00000000..ac2b71e7 --- /dev/null +++ b/migrations/009_backfill_pipeline.sql @@ -0,0 +1,12 @@ +-- +goose Up +-- Re-process accepted lab4 originals; original keys remain durable staging until ready. +INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts) +SELECT id,id,'queued',5 FROM attachments WHERE status='ready' AND variants='[]'::jsonb; +UPDATE attachments SET staging_key=object_key, + object_key='original/'||id::text||'/v1/original',status='queued',updated_at=clock_timestamp() +WHERE id IN (SELECT attachment_id FROM attachment_jobs); +INSERT INTO attachment_outbox(job_id) SELECT id FROM attachment_jobs; + +-- +goose Down +-- Do not erase accepted processing jobs during a downgrade. Schema Down is destructive. +SELECT 1; diff --git a/migrations/010_storage_cleanup.sql b/migrations/010_storage_cleanup.sql new file mode 100644 index 00000000..62cbac50 --- /dev/null +++ b/migrations/010_storage_cleanup.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE attachments ADD COLUMN storage_cleaned_at timestamptz; + +-- +goose Down +ALTER TABLE attachments DROP COLUMN storage_cleaned_at; diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..129feb00 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Only for rendering the measured lab5 research charts. +matplotlib==3.9.4 diff --git a/scripts/benchmark_pipeline.py b/scripts/benchmark_pipeline.py new file mode 100644 index 00000000..191ce11c --- /dev/null +++ b/scripts/benchmark_pipeline.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 track 1 experiment against the isolated Swarm stack. + +Uses synthetic users and PNGs. Changes only worker knobs and admission limits, +then restores their original values. Never deletes volumes or accepted jobs. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import csv +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import struct +import subprocess +import sys +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def service_spec(name): + return json.loads(docker('service', 'inspect', name))[0]['Spec'] + + +def containers(service): + return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service, + '--format', '{{.ID}}').split() + + +def wait_service(name, timeout=120): + deadline, stable, previous = time.monotonic() + timeout, None, None + while time.monotonic() < deadline: + inspected = json.loads(docker('service', 'inspect', name))[0] + spec = inspected['Spec'] + update_state = inspected.get('UpdateStatus', {}).get('State') + check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'), + 'service configuration update failed: ' + name) + expected = spec['Mode']['Replicated']['Replicas'] + ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split() + healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed') + if healthy: + tasks = json.loads(docker('inspect', *ids)) + healthy = all(t['Status']['State'] == 'running' for t in tasks) + if healthy: + states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks])) + healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states) + now, identity = time.monotonic(), frozenset(ids) + if healthy: + if stable is None or identity != previous: + stable = now + if stable is not None and now - stable >= 5: + return + else: + stable = None + previous = identity + time.sleep(.5) + raise AssertionError('service did not stabilize: ' + name) + + +def update_env(service, values): + args = ['service', 'update', '--detach'] + for name, value in values.items(): + args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)] + docker(*args, service) + wait_service(service) + + +def old_env(spec, names): + values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', [])) + return {name: values.get(name) for name in names} + + +def png(width, height): + rng = random.Random(20261110 + width) + def chunk(kind, value): + return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value)) + pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b'')) + + +def percentile(values, fraction): + check(bool(values), 'no observations for percentile') + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] + + +class Experiment: + def __init__(self, args): + self.args = args + self.api = args.stack + '_api' + self.worker = args.stack + '_worker' + self.pg = args.stack + '_postgres' + self.client = Client(args.base_url, timeout=60) + + def sql(self, statement): + ids = containers(self.pg) + check(len(ids) == 1, 'expected one PostgreSQL container') + return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger', + '-At', '-v', 'ON_ERROR_STOP=1', '-c', statement) + + def outstanding(self): + return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')")) + + def sample(self, chat): + # chat is a UUID generated by the API, never an arbitrary SQL fragment. + from uuid import UUID + chat_id = str(UUID(chat['id'])) + query = """SELECT json_build_object( + 'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')), + 'queued', count(*) FILTER (WHERE j.state='queued'), + 'processing', count(*) FILTER (WHERE j.state='processing'), + 'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at) + FILTER (WHERE j.state='queued')),0)) + FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE a.chat_id='""" + chat_id + "'" + result = json.loads(self.sql(query)) + total_rss = 0 + for container in containers(self.worker): + # Only process status counters; no command lines, environment or credentials. + raw = docker('exec', container, 'sh', '-c', + "awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true") + total_rss += int(raw or 0) + result['rss_bytes'] = total_rss + return result + + def upload(self, actor, chat, content): + started = time.monotonic() + response = Client(self.args.base_url, timeout=60).call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png', + [202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + return {'started': started, 'accepted': time.monotonic(), 'status': response['status'], + 'instance': response['headers'].get('X-Instance-Id'), + 'attachment': response['json'] if response['status'] == 202 else None} + + def validate_result(self, actor, attachment, content): + check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}, + 'ready must publish all three variants') + original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor) + check(original['bytes'] == content, 'original changed during pipeline') + for variant in attachment['variants']: + response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor) + check(len(response['bytes']) == variant['size_bytes'], 'variant size differs') + check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs') + + def run_case(self, concurrency, prefetch, width, height): + check(self.outstanding() == 0, 'drain other workloads before research') + update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch}) + alice, bob, _, chat = self.client.bootstrap() + content = png(width, height) + warm = self.upload(alice, chat, content) + check(warm['status'] == 202, 'warmup unexpectedly rejected') + self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout) + records, samples, rejected = [], [], 0 + started, deadline = time.monotonic(), time.monotonic() + self.args.timeout + baseline = self.sample(chat) + with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool: + pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)} + polling = {} + while pending or polling: + check(time.monotonic() < deadline, 'pipeline did not drain before research timeout') + done = {future for future in pending if future.done()} + pending -= done + for future in done: + row = future.result() + if row['status'] == 429: + rejected += 1 + else: + check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot') + polling[row['attachment']['id']] = row + for attachment_id, row in list(polling.items()): + attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id, + actor=bob, schema='Attachment')['json'] + check(attachment['status'] != 'failed', 'research attachment failed') + if attachment['status'] == 'ready': + now = time.monotonic() + row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000, + ready_ms=(now-row['started'])*1000, + after_accept_ms=(now-row['accepted'])*1000) + records.append(row) + del polling[attachment_id] + sample = self.sample(chat) + samples.append({'seconds': round(time.monotonic() - started, 3), **sample}) + if pending or polling: + time.sleep(self.args.interval) + elapsed = time.monotonic() - started + check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix') + check(self.outstanding() == 0, 'comparison workload not drained') + self.validate_result(bob, records[-1]['attachment'], content) + result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height, + 'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(), + 'offered': self.args.requests, 'accepted': len(records), + 'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3), + 'throughput_rps': round(len(records) / elapsed, 3), + 'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3), + 'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3), + 'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3), + 'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3), + 'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples), + 'outstanding_peak': max(s['outstanding'] for s in samples), + 'rss_baseline_bytes': baseline['rss_bytes'], + 'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]), + 'samples': samples, + 'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3), + 'upload_ms': round(r['upload_ms'], 3)} for r in records]} + print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True) + return result + + def overload(self, limit): + check(self.outstanding() == 0, 'drain other workloads before overload test') + docker('service', 'scale', '--detach', self.worker + '=0') + deadline = time.monotonic() + 60 + while containers(self.worker): + check(time.monotonic() < deadline, 'worker did not stop') + time.sleep(.5) + update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit}) + alice, bob, _, chat = self.client.bootstrap() + content = png(320, 200) + with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool: + rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2))) + accepted = [r for r in rows if r['status'] == 202] + rejected = len(rows) - len(accepted) + outstanding = self.outstanding() + check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas') + check(len(accepted) == limit and rejected == limit and outstanding == limit, + 'admission limit did not bound accepted work while worker was stopped') + self.client.message(alice, chat, 'text messages remain available during overload') + docker('service', 'scale', '--detach', self.worker + '=1') + wait_service(self.worker) + started = time.monotonic() + for row in accepted: + attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout) + check(attachment['error_code'] is None, 'accepted work failed after overload') + recovery = time.monotonic() - started + check(self.outstanding() == 0, 'accepted work remained outstanding after recovery') + followup = self.upload(alice, chat, content) + check(followup['status'] == 202, 'API did not resume admission after drain') + self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout) + result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected, + 'observed_api_instances': len({row['instance'] for row in rows}), + 'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted), + 'observed_drain_after_worker_healthy_seconds': round(recovery, 3), + 'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202, + 'text_messages_during_overload': 'PASS'} + print('OVERLOAD ' + json.dumps(result), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085')) + parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5')) + parser.add_argument('--requests', type=int, default=20) + parser.add_argument('--uploaders', type=int, default=8) + parser.add_argument('--interval', type=float, default=.25) + parser.add_argument('--timeout', type=int, default=240) + parser.add_argument('--limit', type=int, default=8) + parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json') + args = parser.parse_args() + check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100, + 'invalid experiment limits') + check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive') + experiment = Experiment(args) + initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api) + original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH')) + original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',)) + original_replicas = initial_worker['Mode']['Replicated']['Replicas'] + check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal') + report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'], + 'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'), + 'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'), + 'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')), + 'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')), + 'architecture': platform.machine(), 'python': platform.python_version()}, + 'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}), + 'initial_settings': {'worker': original_worker_env, 'api': original_api_env}, + 'requests_per_case': args.requests, 'upload_concurrency': args.uploaders, + 'sampling_interval_min_seconds': args.interval, + 'method': {'throughput': 'completed / burst makespan including client upload and polling', + 'ready_latency': 'client POST start to first observed ready; nearest-rank p95', + 'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing', + 'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly', + 'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'}, + 'cases': []} + args.output.parent.mkdir(parents=True, exist_ok=True) + def save(): + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n') + try: + for width, height in ((512, 512), (1536, 1536)): + for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)): + report['cases'].append(experiment.run_case(concurrency, prefetch, width, height)) + save() + report['overload'] = experiment.overload(args.limit) + save() + finally: + # Keep every accepted job recoverable, including when an assertion fails. + try: + if not containers(experiment.worker): + docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas)) + wait_service(experiment.worker) + update_env(experiment.worker, original_worker_env) + finally: + update_env(experiment.api, original_api_env) + report['result'] = 'PASS' + report['restored_initial_settings'] = True + report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat() + save() + summary = args.output.with_suffix('.csv') + keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed', + 'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms', + 'queue_lag_peak_seconds', 'rss_peak_bytes'] + with summary.open('w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore') + writer.writeheader() + writer.writerows(report['cases']) + print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output)) + + +if __name__ == '__main__': + main() diff --git a/scripts/measure_load.py b/scripts/measure_load.py index 82481f07..891b407c 100644 --- a/scripts/measure_load.py +++ b/scripts/measure_load.py @@ -9,7 +9,7 @@ import sys import time ROOT = Path(__file__).resolve().parents[1] -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') def containers(service=None): label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK diff --git a/scripts/measure_uploads.py b/scripts/measure_uploads.py index 7a0de357..07bd16f5 100644 --- a/scripts/measure_uploads.py +++ b/scripts/measure_uploads.py @@ -59,7 +59,7 @@ def main(): elapsed = time.monotonic()-started samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)}) for response in responses: - att = response['json'] + att = c.wait_ready(bob,response['json']) check(att['sha256']==sha,'concurrent upload checksum mismatch') downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed') @@ -77,7 +77,7 @@ def main(): check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit') results['runs'].append(run) print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True) - save('lab4-upload-memory.json',results) + save('lab5-upload-memory.json',results) print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.') diff --git a/scripts/plot_pipeline.py b/scripts/plot_pipeline.py new file mode 100644 index 00000000..d07a880d --- /dev/null +++ b/scripts/plot_pipeline.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Render measured pipeline results; install requirements-research.txt first.""" +import argparse +import json +import os +from pathlib import Path +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input', nargs='?', type=Path, default=Path('evidence/lab5-research.json')) + args = parser.parse_args() + os.environ['MPLCONFIGDIR'] = str(Path(tempfile.gettempdir()) / 'mtusi-lab5-matplotlib') + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + report = json.loads(args.input.read_text()) + assert report.get('result') == 'PASS', 'plot only completed experiments' + cases = report['cases'] + metrics = [ + ('throughput_rps', 'Throughput, images/s', 1), + ('ready_p95_ms', 'p95 POST start → ready, s', .001), + ('queue_lag_peak_seconds', 'Peak queued job age, s', 1), + ('rss_peak_bytes', 'Sampled worker + child RSS, MiB', 1 / 1024**2), + ] + plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False}) + figure, axes = plt.subplots(2, 2, figsize=(10, 7), layout='constrained') + for axis, (metric, label, factor) in zip(axes.flat, metrics): + for width in sorted({case['width'] for case in cases}): + series = sorted([case for case in cases if case['width'] == width], key=lambda c: c['concurrency']) + axis.plot(range(len(series)), [case[metric] * factor for case in series], + marker='o', linewidth=2, label=str(width) + ' × ' + str(series[0]['height'])) + axis.set_xticks(range(len(series)), [str(c['concurrency']) + '/' + str(c['prefetch']) for c in series]) + axis.set_xlabel('Concurrency / prefetch') + axis.set_ylabel(label) + axis.set_ylim(bottom=0) + axis.grid(axis='y', alpha=.25) + axis.legend(title='PNG pixels') + figure.suptitle('Redis Streams image pipeline — finite burst on one host\n' + + str(report['requests_per_case']) + ' images per case; sampled RSS and queue lag', fontsize=13) + for suffix in ('.svg', '.png'): + target = args.input.with_suffix(suffix) + figure.savefig(target, dpi=180, metadata={'Creator': 'Matplotlib'}) + print(target) + plt.close(figure) + + +if __name__ == '__main__': + main() diff --git a/scripts/swarm.py b/scripts/swarm.py index a40d801f..e90e26bb 100644 --- a/scripts/swarm.py +++ b/scripts/swarm.py @@ -10,13 +10,13 @@ import urllib.request ROOT = Path(__file__).resolve().parents[1] os.chdir(ROOT) -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') -API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4') -MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4') -TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') +API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5') +MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5') +TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5') S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z') MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') -PORT = os.environ.get('PORT', '8080') +PORT = os.environ.get('PORT', '18085') SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key', 's3_root_secret_key', 's3_access_key', 's3_secret_key') @@ -71,7 +71,7 @@ def job(kind, image, command, integration=False): # Fixed service name is a deployment lock: concurrent runners fail to create it. args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', - '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m', '--secret', 'source='+STACK+'_postgres_password,target=postgres_password', '--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password'] @@ -113,10 +113,10 @@ def bootstrap_s3(): name = STACK+'_s3-bootstrap' args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', - '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh', '--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh', - '--config', 'source='+STACK+'_s3_policy,target=/policy.json'] + '--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json'] for secret in SECRETS[2:]: args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] docker(*args, MC, '/bootstrap.sh') @@ -131,9 +131,18 @@ def reconcile(): docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:]) +def worker_cli(args): + active=[t for t in tasks('worker') if t['Status']['State']=='running'] + if not active: raise RuntimeError('start worker before using its operator CLI') + docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args) + + def build(): - docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') - docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') + if os.environ.get('SKIP_STORAGE_BUILD') != '1': + docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') + docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') + else: + docker('image','inspect',S3,MC,capture=True) docker('build', '-t', API, '.') docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.') @@ -147,7 +156,7 @@ def up(): raise RuntimeError('run on the Swarm manager') if len(docker('node', 'ls', '-q', capture=True).split()) != 1: raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution') - docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID']) + docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID']) subprocess.run(['./scripts/bootstrap.sh'], check=True) for secret in SECRETS: known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split() @@ -158,6 +167,13 @@ def up(): # Upgrade the shared schema before replacing any running API image. fresh = STACK+'_api' not in existing if not fresh: + # A maintenance barrier prevents old lab4 writers inserting ready-only rows + # after the one-shot async backfill but before the new API image starts. + docker('service','scale','--detach',STACK+'_api=0') + deadline=time.monotonic()+45 + while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip(): + if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations') + time.sleep(.5) job('migrations', MIGRATIONS, ['up']) # Initial deployment keeps API at zero until the one-shot migration succeeds. env = dict(os.environ) @@ -176,12 +192,13 @@ def up(): wait_api() else: update() + wait_service('worker') print('PASS: Swarm stack ready at http://localhost:'+PORT) def restart(): # Force replaces every task while retaining the same Swarm secrets and volumes. - services = ('postgres', 'redis', 'minio', 'api', 'lb') + services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker') previous = {name: {t['ID'] for t in tasks(name)} for name in services} for service in services: docker('service', 'update', '--detach', '--force', STACK+'_'+service) @@ -216,7 +233,9 @@ def update(): state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State') if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')): wait_api() - print('PASS: rolling update completed') + docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker') + wait_service('worker') + print('PASS: rolling update completed (API and worker image refreshed)') return if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'): raise RuntimeError('rolling update: '+state) @@ -226,7 +245,7 @@ def update(): def status(): result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}} - for name in ('lb','api','postgres','redis','minio'): + for name in ('lb','api','worker','postgres','redis','minio'): service = inspect('service', STACK+'_'+name) spec = service['Spec'] template = spec['TaskTemplate'] @@ -248,6 +267,17 @@ def main(): elif action == 'build': build() elif action == 'bootstrap-s3': bootstrap_s3() elif action == 'reconcile': reconcile() + elif action == 'worker-stop': + docker('service','scale','--detach',STACK+'_worker=0') + deadline=time.monotonic()+30 + while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip(): + if time.monotonic()>deadline: raise RuntimeError('worker did not stop') + time.sleep(.2) + elif action == 'worker-start': + docker('service','scale','--detach',STACK+'_worker=1') + wait_service('worker') + elif action == 'failed-jobs': worker_cli(['--failed']) + elif action == 'replay': worker_cli(sys.argv[2:]) elif action == 'wait': wait_api() elif action == 'restart': restart() elif action == 'stop-one': stop_one() diff --git a/stack.yaml b/stack.yaml index da599861..d984126d 100644 --- a/stack.yaml +++ b/stack.yaml @@ -18,7 +18,7 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: limits: {cpus: '1.0', memory: 512M} update_config: {parallelism: 1, order: stop-first} @@ -39,9 +39,9 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: - limits: {cpus: '0.5', memory: 128M} + limits: {cpus: '0.5', memory: 384M} update_config: {parallelism: 1, order: stop-first} minio: image: ${S3_IMAGE:-mtusi-minio:RELEASE.2025-09-07T16-13-09Z} @@ -58,13 +58,13 @@ services: deploy: replicas: 1 placement: - constraints: [node.labels.mtusi_lab4_storage == true] + constraints: [node.labels.mtusi_lab5_storage == true] resources: limits: {cpus: '1.0', memory: 512M} update_config: {parallelism: 1, order: stop-first} api: - image: ${API_IMAGE:-mtusi-messenger:lab4} - environment: + image: ${API_IMAGE:-mtusi-messenger:lab5} + environment: &api_environment INSTANCE_ID: '{{.Task.ID}}' POSTGRES_HOST: postgres POSTGRES_USER: messenger @@ -104,6 +104,27 @@ services: monitor: 15s failure_action: rollback rollback_config: {parallelism: 1, order: start-first} + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + entrypoint: [/usr/local/bin/worker] + environment: + <<: *api_environment + WORKER_CONCURRENCY: '2' + WORKER_PREFETCH: '4' + WORKER_MAX_ATTEMPTS: '5' + WORKER_TIMEOUT_SECONDS: '25' + WORKER_LEASE_SECONDS: '40' + WORKER_PIXEL_BUDGET: '40000000' + secrets: [postgres_password, cursor_key, s3_access_key, s3_secret_key] + networks: [data] + healthcheck: {disable: true} + stop_grace_period: 10s + deploy: + replicas: 1 + resources: + limits: {cpus: '1.0', memory: 1536M} + restart_policy: {condition: any, delay: 1s} + update_config: {parallelism: 1, order: stop-first} lb: image: haproxy:3.2.6-alpine configs: @@ -112,7 +133,7 @@ services: networks: [backend] ports: - target: 8080 - published: ${PORT:-8080} + published: ${PORT:-18085} protocol: tcp mode: host healthcheck: @@ -138,27 +159,29 @@ configs: haproxy: file: ./infra/haproxy.cfg redis: + name: ${STACK_NAME:-mtusi-lab5}_redis_v5 file: ./infra/redis.conf s3_bootstrap: file: ./infra/s3-bootstrap.sh s3_policy: + name: ${STACK_NAME:-mtusi-lab5}_s3_policy_v5 file: ./infra/s3-policy.json secrets: postgres_password: external: true - name: ${STACK_NAME:-mtusi-lab4}_postgres_password + name: ${STACK_NAME:-mtusi-lab5}_postgres_password cursor_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_cursor_key + name: ${STACK_NAME:-mtusi-lab5}_cursor_key s3_root_access_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_root_access_key + name: ${STACK_NAME:-mtusi-lab5}_s3_root_access_key s3_root_secret_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_root_secret_key + name: ${STACK_NAME:-mtusi-lab5}_s3_root_secret_key s3_access_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_access_key + name: ${STACK_NAME:-mtusi-lab5}_s3_access_key s3_secret_key: external: true - name: ${STACK_NAME:-mtusi-lab4}_s3_secret_key + name: ${STACK_NAME:-mtusi-lab5}_s3_secret_key diff --git a/tests/own/README.md b/tests/own/README.md index b446401a..31e10eb5 100644 --- a/tests/own/README.md +++ b/tests/own/README.md @@ -1,20 +1,20 @@ -# Собственные проверки lab4 +# Проверки lab5 -`make test-lab4` проверяет реальные PNG/JPEG, лимиты, MIME, повреждения, права, приватность bucket и отказы. -Тест останавливает именно API-реплику, которая приняла файл, и проверяет скачивание через другую. -`make measure-uploads` запускает десять клиентов на двух размерах PNG и сохраняет замеры cgroup memory. -`make test-reconcile` внедряет ошибки S3/БД и проверяет grace, блокировки, dry-run, повторное исправление и стоимость обхода. -Эти сценарии используют реальный MinIO. Запускайте их последовательно на учебном Swarm-стенде. +`make test` запускает неизменённый публичный smoke lab5. +`make test-own` запускает Go-тесты с race detector. +`make test-integration` проверяет реальные миграции, транзакции, гонки, пагинацию, fencing и версии. +`make test-lab5` проверяет реальные Redis Streams, MinIO и PostgreSQL. +Сценарий проверяет границы staging/COMMIT/publish/objects/COMMIT/ack и восстановление после остановки компонентов. +Он проверяет retry, failed registry, audit replay, process timeout и cleanup. +Проверки JPEG/WebP, EXIF, alpha и малых изображений используют Pillow из `requirements-research.txt`. +Сценарии изменяют только свой стек и синтетические данные. +Запускайте проверки отказов последовательно. -Результаты находятся в `evidence/lab4-*`. Подробная методика приведена в [REPORT.md](../../REPORT.md). +`make test-lab4` и `make test-reconcile` оставлены как совместимые имена для полного `test-lab5`. +Старые `reconcile.py` и `grade4.py` сохраняют исторические сценарии; напрямую их не запускайте. +`make test-persistence` проверяет общий restart; `make test-failover` проверяет остановку одной API-реплики. -Публичные tests/*.py и обязательный контракт сохранены. - -- `make test-own`: Go tests с race detector. Без INTEGRATION_TESTS=1 PostgreSQL-сценарий пропускается. -- `make test-integration`: отдельный Swarm job, Go 1.26.4 и goose. Временная схема, миграции с нуля/повторно/legacy/reset/up, регистрационная гонка, rollback, 10 000 сообщений с одинаковыми timestamps, EXPLAIN и 16 конкурентных запросов одного ключа. -- `make test-grade4`: `swarm_grade4.py`, ровно две API tasks. Проверяет общий ключ через LB, одинаковый результат, 409, границы ключа, область пользователя/чата, права и logout. По очереди останавливает обе исходные tasks с восстановлением, затем выполняет rolling update под нагрузкой. Проверяет историю, отсутствие дублей и повтор исходного ключа после воздействия. Сохраняет `evidence/swarm-grade4.json`. -- `make test-persistence`: перезапуск всех API, PostgreSQL, Redis, MinIO и LB; volumes/secrets сохраняются. -- `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`: публичный сценарий отказа; `make wait` дожидается восстановления двух реплик. -- `make measure-load`: 3s прогрева + 30s, concurrency 4, метрики и Docker stats в `evidence/swarm-*`. - -Сценарии отказов, persistence и rolling update выполняйте **последовательно**, на локальном учебном стенде. Все данные синтетические; пароль и токены в результаты не записываются. Старый `grade4.py` относится к Compose-стенду lab2 и не является entrypoint текущей работы. +`make benchmark` измеряет очередь под перегрузкой для трёх настроек и двух размеров изображений. +`make plot-research` строит графики по фактическому JSON. +Методика и результаты находятся в [RESEARCH.md](../../RESEARCH.md). +Новые доказательства используют имена `evidence/lab5-*`; исходные lab4 evidence считаются историческими. diff --git a/tests/own/helpers.py b/tests/own/helpers.py index ca4dcfb3..d8032414 100644 --- a/tests/own/helpers.py +++ b/tests/own/helpers.py @@ -14,7 +14,7 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') @@ -43,11 +43,11 @@ def reconcile(apply=False, grace='90s'): return json.loads(docker(*args).stdout) -def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=201): +def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=202): from urllib.parse import quote return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+quote(filename, safe=''), expected, actor=actor, raw=content, headers={'Content-Type': mime}, - schema='Attachment' if expected == 201 else None) + schema='Attachment' if expected == 202 else None) def png(width=320, height=200, noisy=False): @@ -72,13 +72,26 @@ def save(name, result): def wait_ready(timeout=90): deadline = time.monotonic()+timeout + stable_since = None + previous = None while time.monotonic() < deadline: try: - Client(timeout=2).call('GET', '/health/ready', schema='Health') - return - except AssertionError: - time.sleep(.5) - raise AssertionError('readiness did not recover') + ids=docker('service','ps','--filter','desired-state=running','-q',STACK+'_api').stdout.decode().split() + wanted=json.loads(docker('service','inspect',STACK+'_api').stdout)[0]['Spec']['Mode']['Replicated']['Replicas'] + check(wanted>0 and len(ids)==wanted,'API count') + ts=json.loads(docker('inspect',*ids).stdout) + check(all(t['Status']['State']=='running' for t in ts),'API task state') + cs=json.loads(docker('inspect',*[t['Status']['ContainerStatus']['ContainerID'] for t in ts]).stdout) + check(all(c['State'].get('Health',{}).get('Status')=='healthy' for c in cs),'API health') + Client(timeout=2).call('GET','/health/ready',schema='Health') + current=frozenset(ids) + if current!=previous: stable_since=time.monotonic() + previous=current + if stable_since is not None and time.monotonic()-stable_since>=5:return + except (AssertionError,subprocess.CalledProcessError,KeyError): + stable_since=None;previous=None + time.sleep(.5) + raise AssertionError('all API replicas and LB did not stabilize') class Probe: diff --git a/tests/own/lab4.py b/tests/own/lab4.py deleted file mode 100644 index 8622723b..00000000 --- a/tests/own/lab4.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -"""Real S3/API negative cases and failures; only synthetic lab data.""" -import hashlib -import http.client -import json -import os -from pathlib import Path -import subprocess -import time -import uuid -from urllib.parse import urlsplit - -from helpers import Client, Probe, ROOT, STACK, check, containers, docker, png, oversized_dimensions, save, sql, upload, wait_ready - - -def main(): - c = Client(timeout=55) - alice, bob, eve, chat = c.bootstrap() - data = png() - result = {'negative_cases': [], 'failures': {}, 'checksums': {}} - for name, body, mime, filename, status in ( - ('oversized_bytes', b'x'*(10*1024*1024+1), 'image/png', 'large.png', 413), - ('damaged_png', data[:50], 'image/png', 'broken.png', 415), - ('corrupt_crc', data[:-16]+b'\xff'*16, 'image/png', 'crc.png', 415), - ('mime_spoof', data, 'image/jpeg', 'fake.jpg', 415), - ('not_image', b'plain text', 'image/png', 'fake.png', 415), - ('unsupported_mime', data, 'application/octet-stream', 'file.png', 415), - ('pixel_budget', oversized_dimensions(), 'image/png', 'large-dimensions.png', 413), - ('unsafe_slash', data, 'image/png', '../file.png', 400), - ('unsafe_backslash', data, 'image/png', 'x\\y.png', 400), - ('unsafe_control', data, 'image/png', 'x\x00.png', 400), - ('empty_filename', data, 'image/png', '', 400), - ): - upload(c, alice, chat, body, filename, mime, status) - result['negative_cases'].append({'case': name, 'status': status}) - for content, filename, mime in ((data, 'изображение.png', 'image/png'), - ((ROOT/'tests/own/fixtures/sample.jpg').read_bytes(), 'photo.jpg', 'image/jpeg')): - att = upload(c, alice, chat, content, filename, mime)['json'] - downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob) - check(downloaded['bytes'] == content, 'original bytes changed') - check(att['sha256'] == hashlib.sha256(content).hexdigest(), 'incorrect SHA256') - check(downloaded['headers'].get_content_type() == mime, 'incorrect content MIME') - result['checksums'][mime] = att['sha256'] - att = upload(c, alice, chat, data)['json'] - for suffix in ('', '/content'): - c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve) - c.call('GET', '/api/v1/attachments/'+str(uuid.uuid4())+suffix, 404, actor=alice) - c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 401) - upload(c, eve, chat, data, expected=404) - c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob) - other = c.call('POST', '/api/v1/chats', 201, actor=alice, - body={'title':'other chat', 'member_ids':[alice['id'], bob['id']]}, schema='Chat')['json'] - path = '/api/v1/chats/'+other['id']+'/messages' - for ids, status in (([att['id']], 404), ([str(uuid.uuid4())], 404), ([att['id'],att['id']], 400)): - c.call('POST', path, status, actor=alice, body={'text':'', 'attachment_ids':ids}, - headers={'Idempotency-Key':uuid.uuid4().hex}) - result['negative_cases'] += [{'case': x, 'status': s} for x,s in ( - ('foreign_attachment',404), ('foreign_chat_link',404), ('unknown_attachment',404), - ('duplicate_ids',400), ('missing_authentication',401), ('unknown_variant',404))] - - # The proxy must stop oversized chunked bodies even without Content-Length. - address = urlsplit(c.base) - conn = http.client.HTTPConnection(address.hostname, address.port, timeout=55) - conn.request('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', - body=iter([b'x'*65536]*161), encode_chunked=True, - headers={'Authorization':'Bearer '+alice['token'], 'Content-Type':'image/png'}) - response = conn.getresponse() - check(response.status == 413, 'chunked upload bypassed byte limit') - response.read(); conn.close() - result['negative_cases'].append({'case':'chunked_oversized', 'status':413}) - - # Disconnect halfway through the declared body. No object/metadata can be published. - before = int(sql('SELECT count(*) FROM attachments;')) - conn = http.client.HTTPConnection(address.hostname, address.port, timeout=5) - conn.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=aborted.png') - conn.putheader('Authorization', 'Bearer '+alice['token']) - conn.putheader('Content-Type', 'image/png') - conn.putheader('Content-Length', str(len(data)+10000)) - conn.endheaders(); conn.send(data[:40]); conn.close() - time.sleep(1) - check(int(sql('SELECT count(*) FROM attachments;')) == before, 'aborted upload published metadata') - leftovers = [docker('exec', container, 'find', '/tmp', '-maxdepth', '1', '-name', 'messenger-upload-*').stdout.decode() - for container in containers('api')] - check(not any(leftovers), 'aborted upload left spool files') - result['failures']['client_disconnect'] = {'metadata_added':0, 'spool_files_remaining':0} - - with Probe() as probe: - key = sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") - raw = docker('exec', containers('minio')[0], 'wget', '-S', '-O', '/dev/null', - 'http://127.0.0.1:9000/attachments/'+key, check_result=False) - check(b'403' in raw.stderr, 'unauthenticated direct S3 GET was not forbidden') - probe.mc('stat', 'local/attachments/'+key) - probe.mc('mb', 'local/forbidden-'+uuid.uuid4().hex[:8], expected=1) - probe.mc('pipe', 'local/attachments/outside-prefix', data=b'x', expected=1) - result['bucket_security'] = {'anonymous_get_status':403, 'create_bucket_denied':True, - 'write_outside_prefix_denied':True, 'console_published':False} - service = json.loads(docker('service', 'inspect', STACK+'_minio').stdout)[0] - check(not service['Spec'].get('EndpointSpec', {}).get('Ports'), 'MinIO publishes an external port') - for _ in range(2): - subprocess.run(['python3', str(ROOT/'scripts/swarm.py'), 'bootstrap-s3'], check=True, - stdout=subprocess.DEVNULL) - check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data, - 'repeated bucket bootstrap changed data') - result['bucket_security']['bootstrap_repeats'] = 2 - - # Stop the replica that accepted THIS upload, then read through another one. - accepted = upload(c,alice,chat,data) - source = accepted['headers']['X-Instance-Id'] - env = dict(os.environ, TARGET_INSTANCE_ID=source) - started = time.monotonic() - subprocess.run(['python3',str(ROOT/'scripts/swarm.py'),'stop-one'],env=env,check=True, - stdout=subprocess.DEVNULL) - deadline = time.monotonic()+10 - while True: - try: - downloaded = Client(timeout=2).call('GET','/api/v1/attachments/'+accepted['json']['id']+'/content',actor=bob) - check(downloaded['headers']['X-Instance-Id']!=source,'stopped upload replica still serves bytes') - check(downloaded['bytes']==data,'cross-replica original changed') - break - except AssertionError: - check(time.monotonic()=4 + def observe(): + try: + return ready() + except (AssertionError,subprocess.CalledProcessError,KeyError,IndexError): + stable[:]=[None,None] + raise + until(observe, timeout=120) + + +def control(service, values=None, replicas=None, force=False): + args = ['service','update','--detach'] + for key,value in (values or {}).items(): + args += ['--env-rm',key] if value is None else ['--env-add',key+'='+str(value)] + if replicas is not None: + args += ['--replicas',str(replicas)] + if force: + args += ['--force'] + docker(*args,STACK+'_'+service) + if replicas is None: + spec = json.loads(docker('service','inspect',STACK+'_'+service).stdout)[0] + replicas = spec['Spec']['Mode']['Replicated']['Replicas'] + settled(service,replicas) + + +def redis(*args): + return json.loads(docker('exec',containers('redis')[0],'redis-cli','--json',*args).stdout) + + +def job(att): + return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='"+att['id']+"';")) + + +def wait_state(att, state, timeout=90): + return until(lambda: job(att), lambda j: j['state']==state, timeout) + + +def variant_check(c, actor, att, original, expected_thumb=(256,160)): + ready = c.wait_ready(actor,att) + check(ready['error_code'] is None, 'ready has error') + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']==original,'original changed') + variants = {v['name']:v for v in ready['variants']} + check(set(variants)=={'thumbnail','crop','optimized'},'incomplete variants') + check((variants['thumbnail']['width'],variants['thumbnail']['height'])==expected_thumb,'thumbnail geometry') + check((variants['crop']['width'],variants['crop']['height'])==(128,128),'crop geometry') + for name,v in variants.items(): + response = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor) + body = response['bytes'] + check(len(body)==v['size_bytes'] and hashlib.sha256(body).hexdigest()==v['sha256'],'variant checksum') + check(response['headers'].get_content_type()==v['content_type'],'variant MIME') + if name=='optimized': + check(body[:4]==b'RIFF' and body[8:12]==b'WEBP','not WebP') + else: + check(jpeg_size(body)==(v['width'],v['height']),'JPEG actual geometry') + return ready + + +def replay(j, expected=True): + result = docker('exec',containers('worker')[0],'/usr/local/bin/worker','--replay',j['id'], + '--operator','synthetic-fault-test','--reason','restore synthetic fixture',check_result=False) + check((result.returncode==0)==expected,'unexpected replay result') + + +def main(): + check(STACK!='mtusi-lab4','refuse source stack') + c = Client(timeout=55) + alice,bob,eve,chat = c.bootstrap() + data = png() + # Inherited lab4 obligations remain public: input validation and access control. + negatives=[] + for name,body,mime,filename,status in ( + ('oversized',b'x'*(10485760+1),'image/png','large.png',413), + ('damaged',data[:45],'image/png','broken.png',415), + ('pixel_budget',oversized_dimensions(),'image/png','huge.png',413), + ('mime_spoof',data,'image/jpeg','fake.jpg',415), + ('filename',data,'image/png','../evil.png',400), + ('unsupported',data,'application/octet-stream','file.png',415)): + upload(c,alice,chat,body,filename,mime,status) + negatives.append({'name':name,'status':status}) + att = upload(c,alice,chat,data)['json'] + variant_check(c,bob,att,data) + c.message(alice,chat,'',attachments=[att['id']]) + c.call('GET','/api/v1/attachments/'+att['id'],404,actor=eve) + c.call('GET','/api/v1/attachments/'+att['id']+'/content',401) + upload(c,eve,chat,data,expected=404) + RESULT['cases']['input_and_acl'] = negatives + # Real small image, alpha and EXIF orientation exercise the actual renderer. + from PIL import Image + for name,im,exif,shape in ( + ('small.png',Image.new('RGBA',(12,8),(255,0,0,0)),None,(12,8)), + ('oriented.jpg',Image.new('RGB',(320,200),(40,120,200)),6,(160,256)), + ('oriented.png',Image.new('RGB',(320,200),(40,120,200)),6,(160,256))): + out=io.BytesIO();kwargs={} + if exif: + tags=Image.Exif();tags[274]=exif;kwargs['exif']=tags + is_jpeg=name.endswith('.jpg') + im.save(out,format='JPEG' if is_jpeg else 'PNG',**kwargs) + body=out.getvalue();item=upload(c,alice,chat,body,name,'image/jpeg' if is_jpeg else 'image/png')['json'] + variant_check(c,bob,item,body,shape) + if name=='small.png': + raw=c.call('GET','/api/v1/attachments/'+item['id']+'/content?variant=thumbnail',actor=bob)['bytes'] + pixel=Image.open(io.BytesIO(raw)).getpixel((0,0));check(min(pixel)>250,'alpha was not white') + RESULT['cases']['small_alpha_exif']='PASS' + print('PASS: input/ACL, three actual formats, geometry, EXIF, alpha',flush=True) + with Probe() as probe: + key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") + anonymous=docker('exec',containers('minio')[0],'wget','-S','-O','/dev/null','http://127.0.0.1:9000/attachments/'+key,check_result=False) + check(b'403' in anonymous.stderr,'bucket public') + probe.mc('pipe','local/attachments/outside-prefix',data=b'x',expected=1) + RESULT['cases']['private_s3']={'anonymous_get':403,'outside_prefix_denied':True} + + # A stage PUT without metadata COMMIT must never produce a 202 or a job. + until(lambda:sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;")=='0',timeout=45) + def stage_keys(): + lines=probe.mc('ls','--recursive','--json','local/attachments/staging/').stdout.decode().splitlines() + return {json.loads(line)['key'] for line in lines if json.loads(line).get('type')=='file'} + old=stage_keys() + filename='db-fault-'+uuid.uuid4().hex+'.png' + sql("CREATE FUNCTION lab5_reject_metadata() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='"+filename+"' THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END; $$; CREATE TRIGGER lab5_reject_metadata BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_reject_metadata();") + try: + upload(c,alice,chat,data,filename=filename,expected=503) + finally: + sql('DROP TRIGGER lab5_reject_metadata ON attachments; DROP FUNCTION lab5_reject_metadata();') + unconfirmed=stage_keys()-old + check(len(unconfirmed)==1,'expected one unconfirmed stage') + check(sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"';")=='0','failed upload published metadata') + orphan=next(iter(unconfirmed)) + time.sleep(6) + check(orphan in stage_keys(),'fresh unconfirmed stage deleted before grace') + RESULT['cases']['stage_before_commit']={'upload_status':503,'metadata':0,'fresh_orphan_preserved':True} + + if '--tail-only' not in sys.argv: + # COMMIT without publish, then API and broker restart with worker absent. + control('worker',replicas=0) + control('api',{'PIPELINE_PAUSE_PUBLISH':'1'}) + item=upload(c,alice,chat,data)['json'] + c.message(alice,chat,'',attachments=[item['id']]) + c.call('GET','/api/v1/attachments/'+item['id']+'/content',409,actor=bob) + j=job(item) + check(sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='"+j['id']+"';")=='t','publish not paused') + control('api',force=True) + control('redis',force=True) + settled('api',2) + control('api',{'PIPELINE_PAUSE_PUBLISH':None}) + until(lambda:sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='"+j['id']+"';")=='t') + before=redis('XRANGE',STREAM,'-','+') + check(any(json.loads(row[1][1])['job_id']==j['id'] for row in before),'job not in stream') + control('redis',force=True) + settled('api',2) + check(any(json.loads(row[1][1])['job_id']==j['id'] for row in redis('XRANGE',STREAM,'-','+')),'broker lost durable job') + control('worker',replicas=1) + variant_check(c,bob,item,data) + RESULT['cases']['commit_publish_and_restarts']={'job_id':j['id'],'api_restart':True,'broker_restarts':2,'link_queued':True} + save();print('PASS: COMMIT/publish gap, queued link, API/broker restarts, worker recovery',flush=True) + + # A killed worker leaves deterministic objects before metadata COMMIT/ack. + control('worker',{'WORKER_TIMEOUT_SECONDS':'5','WORKER_LEASE_SECONDS':'12','PIPELINE_FAILPOINT':'after_objects'}) + item=upload(c,alice,chat,data)['json'] + until(lambda:job(item)['state']=='processing') + until(lambda:probe.mc('stat','local/attachments/original/'+item['id']+'/v1/optimized').returncode==0) + control('worker',{'PIPELINE_FAILPOINT':None}) + variant_check(c,bob,item,data) + j=job(item);check(j['attempts']>=2,'expired processing was not retried') + objects=probe.mc('ls','--recursive','--json','local/attachments/original/'+item['id']+'/').stdout.decode().splitlines() + check(sum(json.loads(x).get('type')=='file' for x in objects)==4,'duplicate objects created') + RESULT['cases']['crash_after_objects']={'attempts':j['attempts'],'objects':4,'fence':j['fence']} + save();print('PASS: crash after objects, expired lease/fencing, same four keys',flush=True) + + # Ready COMMIT before ack: redelivery must leave metadata unchanged. + control('worker',{'PIPELINE_FAILPOINT':'after_commit'}) + item=upload(c,alice,chat,data)['json'];wait_state(item,'ready') + snapshot=c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json'] + check(redis('XPENDING',STREAM,GROUP)[0]>0,'ack happened before crash') + control('worker',{'PIPELINE_FAILPOINT':None}) + until(lambda:redis('XPENDING',STREAM,GROUP)[0]==0,timeout=30) + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']==snapshot,'ready replay changed metadata') + j=job(item) + envelope={'schema_version':1,'job_id':j['id'],'type':'attachment.process.v1','attachment_id':item['id'],'pipeline_version':1,'attempt':1,'created_at':j['created_at']} + redis('XADD',STREAM,'*','job',json.dumps(envelope)) + until(lambda:redis('XLEN',STREAM)==0,timeout=30) + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']==snapshot,'duplicate changed ready') + replay(j,False) + RESULT['cases']['commit_before_ack_and_duplicate']={'ready_unchanged':True,'ready_replay_rejected':True} + save();print('PASS: COMMIT before ack, duplicate no-op and ready replay rejection',flush=True) + + # Temporary storage error, bounded attempts/backoff, failed registry and audit replay. + control('api',{'WORKER_MAX_ATTEMPTS':'2'}) + control('worker',{'PIPELINE_FAILPOINT':'transient'}) + item=upload(c,alice,chat,data)['json'] + retry=until(lambda:job(item),lambda j:j['state']=='queued' and j['attempts']==1) + delay=(dt.datetime.fromisoformat(retry['available_at'].replace('Z','+00:00'))-dt.datetime.fromisoformat(retry['updated_at'].replace('Z','+00:00'))).total_seconds() + check(1.5<=delay<=2.6,'retry backoff/jitter outside configured range') + failed=wait_state(item,'failed',timeout=30) + check(failed['attempts']==2,'retry limit') + meta=c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json'] + check(meta['error_code']=='retry_exhausted','unsafe final code') + c.call('GET','/api/v1/attachments/'+item['id']+'/content',409,actor=bob) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[item['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + registry=json.loads(docker('exec',containers('worker')[0],'/usr/local/bin/worker','--failed').stdout) + check(any(row['job_id']==failed['id'] for row in registry),'failed registry missing') + control('worker',{'PIPELINE_FAILPOINT':None}) + replay(failed) + variant_check(c,bob,item,data) + check(sql("SELECT count(*) FROM attachment_replays WHERE job_id='"+failed['id']+"';")=='1','replay audit missing') + RESULT['cases']['bounded_retry_replay']={'max_attempts':2,'retry_available_at':retry['available_at'],'first_backoff_seconds':round(delay,3),'failed_attempts':failed['attempts'],'audits':1} + save();print('PASS: bounded retry/backoff, failed registry, safe audited replay',flush=True) + + # A permanently corrupted durable stage is not retried endlessly. + control('worker',replicas=0) + item=upload(c,alice,chat,data)['json'] + probe.mc('pipe','local/attachments/staging/'+item['id'],data=b'damaged synthetic stage') + control('worker',replicas=1) + failed=wait_state(item,'failed') + check(failed['attempts']==1,'permanent error retried') + check(c.call('GET','/api/v1/attachments/'+item['id'],actor=bob,schema='Attachment')['json']['error_code']=='invalid_image','permanent classification') + RESULT['cases']['permanent_stage_error']={'attempts':1,'error_code':'invalid_image'} + permanent_att=item + permanent_job=failed + + # Deadline kills the renderer AND its real cwebp child blocked on a FIFO. + control('api',{'WORKER_MAX_ATTEMPTS':'1'}) + control('worker',{'WORKER_TIMEOUT_SECONDS':'2','WORKER_LEASE_SECONDS':'7','PIPELINE_FAILPOINT':'renderer_timeout'}) + item=upload(c,alice,chat,data)['json'];started=time.monotonic() + seen=until(lambda:docker('exec',containers('worker')[0],'ps','-o','pid,args').stdout.decode(),lambda text:'cwebp -quiet' in text,timeout=10) + wait_state(item,'failed',timeout=15) + elapsed=time.monotonic()-started + processes=docker('exec',containers('worker')[0],'ps','-o','pid,args').stdout.decode() + check('cwebp -quiet' not in processes and '/worker render' not in processes,'renderer descendant survived timeout') + RESULT['cases']['renderer_process_group_timeout']={'deadline_seconds':2,'elapsed_seconds':round(elapsed,3),'cwebp_observed':True,'descendants_after_failure':0} + control('worker',{'PIPELINE_FAILPOINT':None,'WORKER_TIMEOUT_SECONDS':'25','WORKER_LEASE_SECONDS':'40'}) + control('api',{'WORKER_MAX_ATTEMPTS':None}) + # Every ready stage is eventually collected; linked final objects remain. + until(lambda:sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;")=='0',timeout=45) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'cleanup removed ready original') + sql("UPDATE attachments SET created_at=created_at-interval '8 days',updated_at=clock_timestamp()-interval '8 days' WHERE id='"+permanent_att['id']+"';") + until(lambda:sql("SELECT storage_cleaned_at IS NOT NULL AND staging_key IS NULL FROM attachments WHERE id='"+permanent_att['id']+"';")=='t',timeout=45) + replay(permanent_job,False) + until(lambda:orphan not in stage_keys(),timeout=100) + RESULT['cases']['staging_cleanup']={'ready_stages_remaining':0,'linked_original_preserved':True,'failed_retention_days':7,'expired_failed_stage_removed':True,'expired_replay_rejected':True,'unconfirmed_orphan_removed':True} + required={'input_and_acl','small_alpha_exif','private_s3','stage_before_commit','commit_publish_and_restarts','crash_after_objects','commit_before_ack_and_duplicate','bounded_retry_replay','permanent_stage_error','renderer_process_group_timeout','staging_cleanup'} + check(required.issubset(RESULT['cases']),'incomplete fault evidence') + RESULT['finished_at']=time.time();save() + print('PASS: all lab5 fault, retry, replay, durability, rendering and cleanup checks',flush=True) + + +if __name__=='__main__': + try: + main() + finally: + # Restore ordinary service configuration even if an assertion fails. + try: + control('worker',{'PIPELINE_FAILPOINT':None,'WORKER_TIMEOUT_SECONDS':'25','WORKER_LEASE_SECONDS':'40'},replicas=1) + control('api',{'WORKER_MAX_ATTEMPTS':None,'PIPELINE_PAUSE_PUBLISH':None}) + finally: + save() diff --git a/tests/own/swarm_grade4.py b/tests/own/swarm_grade4.py index fd37a5bd..5b9c5bd7 100644 --- a/tests/own/swarm_grade4.py +++ b/tests/own/swarm_grade4.py @@ -15,7 +15,7 @@ sys.path.insert(0, str(ROOT/'tests')) from client import Client, Failure, check, run from resilience import history -STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') def docker_json(*args):